scripts/BioBERT_CORD19_predictor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T19:59:44.530658Z",
     "start_time": "2021-05-25T19:59:44.520246Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch\n",
    "from torch.nn import BCEWithLogitsLoss\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "from transformers import AdamW, BertTokenizer, BertModel\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm, trange\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "device\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T19:59:45.402295Z",
     "start_time": "2021-05-25T19:59:45.387047Z"
    }
   },
   "outputs": [],
   "source": [
    "class BioBERTForMultiLabelSequenceClassification(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self, num_labels=2):\n",
    "        super(BioBERTForMultiLabelSequenceClassification, self).__init__()\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        # unzip biobert_model.zip file and use the absolute path to that folder, start training with standard biobert model \n",
    "        self.biobert = BertModel.from_pretrained('path/to/biobert_model.zip')\n",
    "        \n",
    "        self.classifier = torch.nn.Linear(768, num_labels)\n",
    "        \n",
    "        torch.nn.init.xavier_normal_(self.classifier.weight)\n",
    "    \n",
    "    def forward(self, input_ids, token_type_ids=None,attention_mask=None, labels=None):\n",
    "        cont_reps, _ = self.biobert(input_ids, attention_mask = attention_mask) \n",
    "        cls_rep = cont_reps[:, 0]\n",
    "        logits = self.classifier(cls_rep)\n",
    "\n",
    "        if labels is not None:\n",
    "            loss_fct = BCEWithLogitsLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_labels),labels.view(-1, self.num_labels))\n",
    "            return loss \n",
    "        else:\n",
    "            return logits \n",
    "        \n",
    "    def freeze_bert_decoder(self):\n",
    "        for param in self.xlnet.parameters():\n",
    "            param.requires_grad = False\n",
    "\n",
    "    def unfreeze_bert_decoder(self):\n",
    "        for param in self.xlnet.parameters():\n",
    "            param.requires_grad = True\n",
    "\n",
    "def train(model, num_epochs,\\\n",
    "          optimizer,\\\n",
    "          train_dataloader, valid_dataloader,\\\n",
    "          model_save_path,\\\n",
    "          train_loss_set=[], valid_loss_set = [],\\\n",
    "          lowest_eval_loss=None, start_epoch=0,\\\n",
    "          device=\"cpu\"\n",
    "          ):\n",
    "    \n",
    "    model.to(device)\n",
    "    \n",
    "    for i in trange(num_epochs, desc=\"Epoch\"):\n",
    "        \n",
    "        actual_epoch = start_epoch + i\n",
    "        \n",
    "        model.train()\n",
    "\n",
    "        tr_loss = 0\n",
    "        num_train_samples = 0\n",
    "\n",
    "        for step, batch in enumerate(train_dataloader):\n",
    "            \n",
    "            print('step: {}'.format(step), end='\\r')\n",
    "            \n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            \n",
    "            b_input_ids, b_input_mask, b_labels = batch\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
    "            \n",
    "            tr_loss += loss.item()\n",
    "            \n",
    "            num_train_samples += b_labels.size(0)\n",
    "            \n",
    "            loss.backward()\n",
    "            \n",
    "            optimizer.step()\n",
    "        \n",
    "        epoch_train_loss = tr_loss/num_train_samples\n",
    "        train_loss_set.append(epoch_train_loss)\n",
    "\n",
    "        print(\"Train loss: {}\".format(epoch_train_loss))\n",
    "\n",
    "        model.eval()\n",
    "       \n",
    "        eval_loss = 0\n",
    "        num_eval_samples = 0\n",
    "\n",
    "        for batch in valid_dataloader:\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            b_input_ids, b_input_mask, b_labels = batch\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\n",
    "                eval_loss += loss.item()\n",
    "                num_eval_samples += b_labels.size(0)\n",
    "        \n",
    "        epoch_eval_loss = eval_loss/num_eval_samples\n",
    "        valid_loss_set.append(epoch_eval_loss)\n",
    "\n",
    "        print(\"Valid loss: {}\".format(epoch_eval_loss))\n",
    "        \n",
    "        if lowest_eval_loss == None:\n",
    "            lowest_eval_loss = epoch_eval_loss\n",
    "            \n",
    "            save_model(model, model_save_path + 'biobert{}.dat'.format(i) , actual_epoch,lowest_eval_loss, train_loss_set, valid_loss_set)\n",
    "        \n",
    "         \n",
    "        else:\n",
    "            \n",
    "            if epoch_eval_loss < lowest_eval_loss:\n",
    "                lowest_eval_loss = epoch_eval_loss\n",
    "                \n",
    "                save_model(model, model_save_path + 'biobert{}.dat'.format(i), actual_epoch,\\\n",
    "                       lowest_eval_loss, train_loss_set, valid_loss_set)\n",
    "        print(\"\\n\")\n",
    "    \n",
    "   \n",
    "    return model, train_loss_set, valid_loss_set\n",
    "\n",
    "\n",
    "def save_model(model, save_path, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist):\n",
    "    \n",
    "    model_to_save = model.module if hasattr(model, 'module') else model\n",
    "    \n",
    "    checkpoint = {'epochs': epochs, \\\n",
    "                    'lowest_eval_loss': lowest_eval_loss,\\\n",
    "                    'state_dict': model_to_save.state_dict(),\\\n",
    "                    'train_loss_hist': train_loss_hist,\\\n",
    "                    'valid_loss_hist': valid_loss_hist\n",
    "                   }\n",
    "    \n",
    "    torch.save(checkpoint, save_path)\n",
    "    \n",
    "    print(\"Saving model at epoch {} with validation loss of {}\".format(epochs,\\\n",
    "                                                                     lowest_eval_loss))\n",
    "    return\n",
    "\n",
    "def load_model(save_path):\n",
    "    checkpoint = torch.load(save_path)\n",
    "    model_state_dict = checkpoint['state_dict']\n",
    "    model = BioBERTForMultiLabelSequenceClassification(num_labels=model_state_dict[\"classifier.weight\"].size()[0])\n",
    "    \n",
    "    model.load_state_dict(model_state_dict)\n",
    "\n",
    "    epochs = checkpoint[\"epochs\"]\n",
    "    lowest_eval_loss = checkpoint[\"lowest_eval_loss\"]\n",
    "    train_loss_hist = checkpoint[\"train_loss_hist\"]\n",
    "    valid_loss_hist = checkpoint[\"valid_loss_hist\"]\n",
    "\n",
    "    return model, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T19:59:45.902009Z",
     "start_time": "2021-05-25T19:59:45.893291Z"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize_inputs(text_list, tokenizer, num_embeddings=512):\n",
    "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t)[:num_embeddings-2], text_list))\n",
    "    input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
    "    input_ids = [tokenizer.build_inputs_with_special_tokens(x) for x in input_ids]\n",
    "    input_ids = pad_sequences(input_ids, maxlen=num_embeddings, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
    "    return input_ids\n",
    "\n",
    "def create_attn_masks(input_ids):\n",
    "    attention_masks = []\n",
    "\n",
    "    for seq in input_ids:\n",
    "        seq_mask = [float(i>0) for i in seq]\n",
    "        attention_masks.append(seq_mask)\n",
    "    return attention_masks\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make predictions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T19:59:50.279969Z",
     "start_time": "2021-05-25T19:59:46.900175Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0525 15:59:48.656962 140173946726144 configuration_utils.py:254] loading configuration file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/config.json\n",
      "I0525 15:59:48.657633 140173946726144 configuration_utils.py:292] Model config BertConfig {\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0525 15:59:48.658384 140173946726144 modeling_utils.py:459] loading weights file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "# LOAD MODEL \n",
    "path = 'path/to/biobert_epistemonikos_finetuned.dat'\n",
    "\n",
    "model, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist = load_model(path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read & process episte test set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T19:59:50.305578Z",
     "start_time": "2021-05-25T19:59:50.281128Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0525 15:59:50.281776 140173946726144 tokenization_utils.py:417] Model name '/mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch' not found in model shortcut name list (bert-base-cased, bert-base-finnish-uncased-v1, bert-base-german-cased, bert-base-german-dbmdz-uncased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-cased-whole-word-masking-finetuned-squad, bert-large-cased, bert-base-dutch-cased, bert-base-chinese, bert-large-uncased, bert-base-multilingual-uncased, bert-large-uncased-whole-word-masking-finetuned-squad, bert-base-german-dbmdz-cased, bert-base-multilingual-cased, bert-base-uncased, bert-base-finnish-cased-v1, bert-base-cased-finetuned-mrpc). Assuming '/mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "I0525 15:59:50.282446 140173946726144 tokenization_utils.py:446] Didn't find file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/added_tokens.json. We won't load it.\n",
      "I0525 15:59:50.282916 140173946726144 tokenization_utils.py:499] loading file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/special_tokens_map.json\n",
      "I0525 15:59:50.283359 140173946726144 tokenization_utils.py:499] loading file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/vocab.txt\n",
      "I0525 15:59:50.283720 140173946726144 tokenization_utils.py:499] loading file /mnt/data2/BERT_Embeddings/models/biobert_v1.1_pubmed_pytorch/tokenizer_config.json\n",
      "I0525 15:59:50.284078 140173946726144 tokenization_utils.py:499] loading file None\n"
     ]
    }
   ],
   "source": [
    "# unzip biobert_model.zip file and use the absolute path to that folder\n",
    "tokenizer = BertTokenizer.from_pretrained('path/to/biobert_model', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:01:09.938968Z",
     "start_time": "2021-05-25T19:59:50.306927Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cord_uid</th>\n",
       "      <th>sha</th>\n",
       "      <th>source_x</th>\n",
       "      <th>title</th>\n",
       "      <th>doi</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>pubmed_id</th>\n",
       "      <th>license</th>\n",
       "      <th>abstract</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>...</th>\n",
       "      <th>who_covidence_id</th>\n",
       "      <th>arxiv_id</th>\n",
       "      <th>pdf_json_files</th>\n",
       "      <th>pmc_json_files</th>\n",
       "      <th>url</th>\n",
       "      <th>s2_id</th>\n",
       "      <th>label</th>\n",
       "      <th>document</th>\n",
       "      <th>features</th>\n",
       "      <th>masks</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pubmed_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32572175</th>\n",
       "      <td>jq2em42q</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>The end of social confinement and COVID-19 re-...</td>\n",
       "      <td>10.1038/s41562-020-0908-8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32572175</td>\n",
       "      <td>unk</td>\n",
       "      <td>The lack of effective pharmaceutical intervent...</td>\n",
       "      <td>2020-06-22</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1038/s41562-020-0908-8; htt...</td>\n",
       "      <td>219958669.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>The end of social confinement and COVID-19 re-...</td>\n",
       "      <td>[101, 1103, 1322, 1104, 1934, 24478, 1105, 188...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33079612</th>\n",
       "      <td>zcxwnvy2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>The Missing Link in the Covid-19 Vaccine Race.</td>\n",
       "      <td>10.1080/21645515.2020.1831859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33079612</td>\n",
       "      <td>unk</td>\n",
       "      <td>Operation Warp Speed and global vaccine resear...</td>\n",
       "      <td>2020-10-20</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/21645515.2020.1831859;...</td>\n",
       "      <td>224826061.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>The Missing Link in the Covid-19 Vaccine Race....</td>\n",
       "      <td>[101, 1103, 3764, 5088, 1107, 1103, 1884, 1831...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33565705</th>\n",
       "      <td>izqvn6nr</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Donor To Recipient Transmission Of SARS-CoV-2 ...</td>\n",
       "      <td>10.1111/ajt.16532</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33565705</td>\n",
       "      <td>unk</td>\n",
       "      <td>We describe a case of proven transmission of S...</td>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/ajt.16532; https://www...</td>\n",
       "      <td>231872023.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Donor To Recipient Transmission Of SARS-CoV-2 ...</td>\n",
       "      <td>[101, 16667, 1106, 7668, 6580, 1104, 21718, 17...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33479162</th>\n",
       "      <td>r17yloy3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Neonates Born to Mothers With COVID-19: Data F...</td>\n",
       "      <td>10.1542/peds.2020-015065</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33479162</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVES To describe neonatal and maternal c...</td>\n",
       "      <td>2021-01-21</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1542/peds.2020-015065; http...</td>\n",
       "      <td>231676588.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Neonates Born to Mothers With COVID-19: Data F...</td>\n",
       "      <td>[101, 24762, 5430, 1255, 1106, 12694, 1114, 18...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33705595</th>\n",
       "      <td>20p9lr62</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>New targets for drug design: Importance of nsp...</td>\n",
       "      <td>10.1111/febs.15815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33705595</td>\n",
       "      <td>unk</td>\n",
       "      <td>SARS-CoV-2 virus has triggered a global pandem...</td>\n",
       "      <td>2021-03-11</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/febs.15815; https://ww...</td>\n",
       "      <td>232208657.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>New targets for drug design: Importance of nsp...</td>\n",
       "      <td>[101, 1207, 7539, 1111, 3850, 1902, 131, 4495,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33771818</th>\n",
       "      <td>9asnam5x</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Establishing a COVID-19 treatment centre in Is...</td>\n",
       "      <td>10.1136/emermed-2020-209639</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33771818</td>\n",
       "      <td>unk</td>\n",
       "      <td>Anticipating the need for a COVID-19 treatment...</td>\n",
       "      <td>2021-03-26</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1136/emermed-2020-209639; h...</td>\n",
       "      <td>232369853.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Establishing a COVID-19 treatment centre in Is...</td>\n",
       "      <td>[101, 7046, 170, 1884, 18312, 118, 1627, 3252,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32890322</th>\n",
       "      <td>y4p2ogrz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Guidelines for Ultrasound in the Radiology Dep...</td>\n",
       "      <td>10.1097/ruq.0000000000000526</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32890322</td>\n",
       "      <td>unk</td>\n",
       "      <td>The coronavirus disease 2019 is caused by the ...</td>\n",
       "      <td>2020-09-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1097/ruq.0000000000000526; ...</td>\n",
       "      <td>221511545.0</td>\n",
       "      <td>broad-synthesis</td>\n",
       "      <td>Guidelines for Ultrasound in the Radiology Dep...</td>\n",
       "      <td>[101, 13112, 1111, 18737, 22909, 1107, 1103, 2...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33381888</th>\n",
       "      <td>9zi08xl7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Antiphospholipid antibodies and thrombosis in ...</td>\n",
       "      <td>10.1002/art.41634</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33381888</td>\n",
       "      <td>unk</td>\n",
       "      <td>We read with great interest the study by Berti...</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1002/art.41634; https://www...</td>\n",
       "      <td>229928885.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Antiphospholipid antibodies and thrombosis in ...</td>\n",
       "      <td>[101, 2848, 7880, 2155, 7880, 11014, 25786, 26...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32628408</th>\n",
       "      <td>c1xd4mtz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Nonoperating room anesthesia for patients with...</td>\n",
       "      <td>10.1097/aco.0000000000000890</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32628408</td>\n",
       "      <td>unk</td>\n",
       "      <td>PURPOSE OF REVIEW To provide aids to deal with...</td>\n",
       "      <td>2020-08-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1097/aco.0000000000000890; ...</td>\n",
       "      <td>220388324.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Nonoperating room anesthesia for patients with...</td>\n",
       "      <td>[101, 1664, 19807, 3798, 1395, 1126, 2556, 273...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33179469</th>\n",
       "      <td>ws1xhm52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Surgical management protocol during the COVID-...</td>\n",
       "      <td>10.23736/s0026-4733.20.08632-0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33179469</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND In the surgical scenario, the sever...</td>\n",
       "      <td>2020-11-11</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.23736/s0026-4733.20.08632-0...</td>\n",
       "      <td>226310156.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Surgical management protocol during the COVID-...</td>\n",
       "      <td>[101, 13467, 2635, 11309, 1219, 1103, 1884, 18...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33305712</th>\n",
       "      <td>ewhqdywx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>COVID-19 Severity: Lung-Heart Interplay.</td>\n",
       "      <td>10.2174/1573403x16999201210200614</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33305712</td>\n",
       "      <td>unk</td>\n",
       "      <td>In December 2019, a novel COVID-19 caused by S...</td>\n",
       "      <td>2020-12-10</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.2174/1573403x16999201210200...</td>\n",
       "      <td>228100886.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>COVID-19 Severity: Lung-Heart Interplay. In De...</td>\n",
       "      <td>[101, 1884, 18312, 118, 1627, 20997, 131, 1309...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32920815</th>\n",
       "      <td>0wu2pl1r</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Novel coronavirus(Covid19), Its renal manifest...</td>\n",
       "      <td>10.22037/uj.v16i7.6431</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32920815</td>\n",
       "      <td>unk</td>\n",
       "      <td>No abstract.</td>\n",
       "      <td>2020-09-13</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.22037/uj.v16i7.6431; https:...</td>\n",
       "      <td>221674559.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Novel coronavirus(Covid19), Its renal manifest...</td>\n",
       "      <td>[101, 2281, 1884, 15789, 27608, 113, 1884, 183...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32657602</th>\n",
       "      <td>a980qiu9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>The COVID-19 Outbreak in Iran; The First Patie...</td>\n",
       "      <td>10.34172/aim.2020.48</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32657602</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>The novel coronavirus, formerly named as 2019 ...</td>\n",
       "      <td>2020-07-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.34172/aim.2020.48; https://...</td>\n",
       "      <td>220500434.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>The COVID-19 Outbreak in Iran; The First Patie...</td>\n",
       "      <td>[101, 1103, 1884, 18312, 118, 1627, 8010, 1107...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32601125</th>\n",
       "      <td>s7u0ct2l</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Potential role of endothelial cell surface ect...</td>\n",
       "      <td>10.7861/clinmed.2020-0252</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32601125</td>\n",
       "      <td>unk</td>\n",
       "      <td>The novel coronavirus infectious disease (COVI...</td>\n",
       "      <td>2020-06-29</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.7861/clinmed.2020-0252; htt...</td>\n",
       "      <td>220268336.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Potential role of endothelial cell surface ect...</td>\n",
       "      <td>[101, 3209, 1648, 1104, 1322, 12858, 21091, 13...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33620014</th>\n",
       "      <td>203ib111</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Convalescent Plasma and Hyperimmune Globulin T...</td>\n",
       "      <td>10.1080/1744666x.2021.1894927</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33620014</td>\n",
       "      <td>unk</td>\n",
       "      <td>INTRODUCTION Severe acute respiratory syndrome...</td>\n",
       "      <td>2021-02-23</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/1744666x.2021.1894927;...</td>\n",
       "      <td>232019282.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Convalescent Plasma and Hyperimmune Globulin T...</td>\n",
       "      <td>[101, 14255, 18236, 23193, 13441, 1105, 177, 2...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33533477</th>\n",
       "      <td>97s2ieqi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Nitric oxide and Covid-19: dose, timing and ho...</td>\n",
       "      <td>10.1111/aas.13788</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33533477</td>\n",
       "      <td>unk</td>\n",
       "      <td>Nitric oxide (NO) for inhalation has been a cl...</td>\n",
       "      <td>2021-02-03</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/aas.13788; https://www...</td>\n",
       "      <td>231789608.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Nitric oxide and Covid-19: dose, timing and ho...</td>\n",
       "      <td>[101, 11437, 11048, 17151, 1105, 1884, 18312, ...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33290467</th>\n",
       "      <td>4yalw4qa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Introductory lecture: air quality in megacities.</td>\n",
       "      <td>10.1039/d0fd00123f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33290467</td>\n",
       "      <td>cc-by-nc</td>\n",
       "      <td>Urbanization is an ongoing global phenomenon a...</td>\n",
       "      <td>2020-12-08</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1039/d0fd00123f; https://ww...</td>\n",
       "      <td>228081191.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Introductory lecture: air quality in megacitie...</td>\n",
       "      <td>[101, 27553, 26739, 10309, 131, 1586, 3068, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33645461</th>\n",
       "      <td>jc38sall</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Pneumocystis jirovecii pneumonia in an immunoc...</td>\n",
       "      <td>10.1080/23744235.2021.1890331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33645461</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND Several cases of invasive fungal di...</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/23744235.2021.1890331;...</td>\n",
       "      <td>232079882.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Pneumocystis jirovecii pneumonia in an immunoc...</td>\n",
       "      <td>[101, 185, 1673, 1818, 13335, 6834, 6620, 2322...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33004394</th>\n",
       "      <td>qvil8upe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Nutrition screening and assessment tools for p...</td>\n",
       "      <td>10.1136/bmjopen-2020-037844</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33004394</td>\n",
       "      <td>cc-by-nc</td>\n",
       "      <td>INTRODUCTION Nutritional challenges are common...</td>\n",
       "      <td>2020-10-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1136/bmjopen-2020-037844; h...</td>\n",
       "      <td>222145118.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Nutrition screening and assessment tools for p...</td>\n",
       "      <td>[101, 20121, 11954, 1105, 8670, 5537, 1111, 44...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33739446</th>\n",
       "      <td>iudemqdy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Left-behind children's social adjustment and r...</td>\n",
       "      <td>10.1002/ijop.12754</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33739446</td>\n",
       "      <td>unk</td>\n",
       "      <td>Using data collected from two provinces in Chi...</td>\n",
       "      <td>2021-03-19</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1002/ijop.12754; https://ww...</td>\n",
       "      <td>232305011.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Left-behind children's social adjustment and r...</td>\n",
       "      <td>[101, 1286, 118, 1481, 1482, 112, 188, 1934, 2...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33557718</th>\n",
       "      <td>6paiox0q</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Social work leadership in a medical school: A ...</td>\n",
       "      <td>10.1080/00981389.2021.1885567</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33557718</td>\n",
       "      <td>unk</td>\n",
       "      <td>The COVID-19 pandemic has exposed the systemic...</td>\n",
       "      <td>2021-02-09</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/00981389.2021.1885567;...</td>\n",
       "      <td>231864146.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Social work leadership in a medical school: A ...</td>\n",
       "      <td>[101, 1934, 1250, 3645, 1107, 170, 2657, 1278,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32459567</th>\n",
       "      <td>wyln1jpr</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Use of systemic therapies for psoriasis in the...</td>\n",
       "      <td>10.1080/09546634.2020.1775774</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32459567</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND In late 2019 a viral pneumonia bega...</td>\n",
       "      <td>2020-05-27</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/09546634.2020.1775774;...</td>\n",
       "      <td>218975132.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Use of systemic therapies for psoriasis in the...</td>\n",
       "      <td>[101, 1329, 1104, 27410, 1103, 14543, 1905, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33615885</th>\n",
       "      <td>fy46t6hm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Do recipients of the first dose of the Pfizer/...</td>\n",
       "      <td>10.1177/0025817221990520</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33615885</td>\n",
       "      <td>unk</td>\n",
       "      <td>The Covid-19 pandemic in the UK has been great...</td>\n",
       "      <td>2021-02-22</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/0025817221990520; http...</td>\n",
       "      <td>231988376.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Do recipients of the first dose of the Pfizer/...</td>\n",
       "      <td>[101, 1202, 18509, 1104, 1103, 1148, 13753, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33315291</th>\n",
       "      <td>upkafkkf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Does inflammation link stress to poor COVID-19...</td>\n",
       "      <td>10.1002/smi.3017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33315291</td>\n",
       "      <td>unk</td>\n",
       "      <td>Coronavirus disease 2019 (COVID-19) continues ...</td>\n",
       "      <td>2020-12-14</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1002/smi.3017; https://www....</td>\n",
       "      <td>229175021.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Does inflammation link stress to poor COVID-19...</td>\n",
       "      <td>[101, 1674, 24970, 5088, 6600, 1106, 2869, 188...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32866992</th>\n",
       "      <td>ni5oyytr</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Paucity of COVID-19 Dermatology Literature fro...</td>\n",
       "      <td>10.1111/bjd.19505</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32866992</td>\n",
       "      <td>unk</td>\n",
       "      <td>Since emerging in December 2019, COVID-19 has ...</td>\n",
       "      <td>2020-08-31</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/bjd.19505; https://www...</td>\n",
       "      <td>221405192.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Paucity of COVID-19 Dermatology Literature fro...</td>\n",
       "      <td>[101, 185, 3984, 9041, 1104, 1884, 18312, 118,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28958212</th>\n",
       "      <td>pip5ybqd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Effectiveness of remote feedback on physical a...</td>\n",
       "      <td>10.1177/1357633x17733772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28958212</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVES The objective of this systematic re...</td>\n",
       "      <td>2019</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/1357633x17733772; http...</td>\n",
       "      <td>35565293.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Effectiveness of remote feedback on physical a...</td>\n",
       "      <td>[101, 12949, 1104, 6456, 13032, 1113, 2952, 32...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33647224</th>\n",
       "      <td>9hajj5cz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Results of preoperative screening for COVID-19...</td>\n",
       "      <td>10.1080/21548331.2021.1898158</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33647224</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVES Many hospitals recently instituted ...</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/21548331.2021.1898158;...</td>\n",
       "      <td>232091699.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Results of preoperative screening for COVID-19...</td>\n",
       "      <td>[101, 2686, 1104, 3073, 19807, 5838, 11954, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33770334</th>\n",
       "      <td>7r5cz2qc</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>The Impact of COVID-19 on Access to HIV Preven...</td>\n",
       "      <td>10.1007/s10900-021-00979-0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33770334</td>\n",
       "      <td>unk</td>\n",
       "      <td>The COVID-19 pandemic has necessitated restric...</td>\n",
       "      <td>2021-03-26</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1007/s10900-021-00979-0; ht...</td>\n",
       "      <td>232371065.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>The Impact of COVID-19 on Access to HIV Preven...</td>\n",
       "      <td>[101, 1103, 3772, 1104, 1884, 18312, 118, 1627...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33569794</th>\n",
       "      <td>mqvho63h</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>New thing, The Impact of the Emergence of COVI...</td>\n",
       "      <td>10.1002/pd.5902</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33569794</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVE We conducted a study to examine the ...</td>\n",
       "      <td>2021-02-10</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1002/pd.5902; https://www.n...</td>\n",
       "      <td>231882629.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>New thing, The Impact of the Emergence of COVI...</td>\n",
       "      <td>[101, 1207, 1645, 117, 1103, 3772, 1104, 1103,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33368849</th>\n",
       "      <td>kyq4ek3s</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Perceived stress and social support in a large...</td>\n",
       "      <td>10.1111/ene.14697</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33368849</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND The COVID-19 emergency may cause po...</td>\n",
       "      <td>2020-12-23</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/ene.14697; https://www...</td>\n",
       "      <td>229691636.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Perceived stress and social support in a large...</td>\n",
       "      <td>[101, 8367, 6600, 1105, 1934, 1619, 1107, 170,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33283686</th>\n",
       "      <td>hickh15h</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Effect of coronaphobia on smoking habits.</td>\n",
       "      <td>10.1080/10550887.2020.1849950</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33283686</td>\n",
       "      <td>unk</td>\n",
       "      <td>INTRODUCTION AND OBJECTIVE Coronavirus disease...</td>\n",
       "      <td>2020-12-07</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/10550887.2020.1849950;...</td>\n",
       "      <td>227524516.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Effect of coronaphobia on smoking habits. INTR...</td>\n",
       "      <td>[101, 2629, 1104, 1884, 15789, 7880, 23145, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33730407</th>\n",
       "      <td>4siasmyn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>A simple, safe and sensitive method for SARS-C...</td>\n",
       "      <td>10.1111/apm.13123</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33730407</td>\n",
       "      <td>unk</td>\n",
       "      <td>The SARS-CoV-2 pandemic has created an urgent ...</td>\n",
       "      <td>2021-03-17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/apm.13123; https://www...</td>\n",
       "      <td>232300402.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>A simple, safe and sensitive method for SARS-C...</td>\n",
       "      <td>[101, 170, 3014, 117, 2914, 1105, 7246, 3442, ...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28840782</th>\n",
       "      <td>gi3oh9o8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Comparison of three blood transfusion guidelin...</td>\n",
       "      <td>10.1177/1098612x17727233</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28840782</td>\n",
       "      <td>unk</td>\n",
       "      <td>Objectives The increased demand for animal blo...</td>\n",
       "      <td>2018</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/1098612x17727233; http...</td>\n",
       "      <td>40081141.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Comparison of three blood transfusion guidelin...</td>\n",
       "      <td>[101, 7577, 1104, 1210, 1892, 14715, 17149, 13...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32419599</th>\n",
       "      <td>i9nr0zdi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Are there positive lessons for Italy's NHS res...</td>\n",
       "      <td>10.1177/0025817220923698</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32419599</td>\n",
       "      <td>unk</td>\n",
       "      <td>The authors evaluate the importance of prevent...</td>\n",
       "      <td>2020-05-18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/0025817220923698; http...</td>\n",
       "      <td>218677522.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Are there positive lessons for Italy's NHS res...</td>\n",
       "      <td>[101, 1132, 1175, 3112, 8497, 1111, 1122, 1744...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33526471</th>\n",
       "      <td>7douonww</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>A Novel Soluble ACE2 Variant with Prolonged Du...</td>\n",
       "      <td>10.1681/asn.2020101537</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33526471</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND There is an urgent need for approac...</td>\n",
       "      <td>2021-02-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1681/asn.2020101537; https:...</td>\n",
       "      <td>231760153.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>A Novel Soluble ACE2 Variant with Prolonged Du...</td>\n",
       "      <td>[101, 170, 2281, 28043, 20839, 1477, 8120, 111...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31920368</th>\n",
       "      <td>kl2766oe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Nonpharmacological Interventions Addressing Pa...</td>\n",
       "      <td>10.2147/jpr.s216807</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31920368</td>\n",
       "      <td>cc-by-nc</td>\n",
       "      <td>Purpose Children and adolescents with primary ...</td>\n",
       "      <td>2019</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.2147/jpr.s216807; https://w...</td>\n",
       "      <td>210118378.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Nonpharmacological Interventions Addressing Pa...</td>\n",
       "      <td>[101, 1664, 20695, 10841, 23388, 22496, 13735,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33141520</th>\n",
       "      <td>aafmh4h9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Proteomic Biomarkers to Guide Stratification f...</td>\n",
       "      <td>10.1002/pmic.202000229</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33141520</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>Proteins and peptides serve as biomarkers in t...</td>\n",
       "      <td>2020-11-03</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1002/pmic.202000229; https:...</td>\n",
       "      <td>226244341.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Proteomic Biomarkers to Guide Stratification f...</td>\n",
       "      <td>[101, 5250, 1566, 18882, 1665, 25128, 8519, 14...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33085086</th>\n",
       "      <td>7gceby4t</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Increased occurrence of hand eczema in young c...</td>\n",
       "      <td>10.1111/cod.13727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33085086</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND During the first wave of the COVID-...</td>\n",
       "      <td>2020-10-21</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/cod.13727; https://www...</td>\n",
       "      <td>224829028.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Increased occurrence of hand eczema in young c...</td>\n",
       "      <td>[101, 2569, 15299, 1104, 1289, 174, 18587, 144...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33744954</th>\n",
       "      <td>tqil35of</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Anti-SARS-CoV2 antibody responses in serum and...</td>\n",
       "      <td>10.1093/infdis/jiab153</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33744954</td>\n",
       "      <td>unk</td>\n",
       "      <td>Antibody responses to SARS-CoV-2 in serum and ...</td>\n",
       "      <td>2021-03-21</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1093/infdis/jiab153; https:...</td>\n",
       "      <td>232310704.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Anti-SARS-CoV2 antibody responses in serum and...</td>\n",
       "      <td>[101, 2848, 118, 21718, 1733, 118, 1884, 1964,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32571147</th>\n",
       "      <td>8mcrvtp2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>The Difficult Airway and Aerosol-Generating Pr...</td>\n",
       "      <td>10.1177/0194599820936615</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32571147</td>\n",
       "      <td>unk</td>\n",
       "      <td>The impact of the COVID-19 pandemic on otolary...</td>\n",
       "      <td>2020-06-23</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/0194599820936615; http...</td>\n",
       "      <td>219991330.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>The Difficult Airway and Aerosol-Generating Pr...</td>\n",
       "      <td>[101, 1103, 2846, 1586, 2787, 1105, 170, 10771...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33123349</th>\n",
       "      <td>c776npf0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>In vitro efficacy of a copper iodine complex P...</td>\n",
       "      <td>10.12688/f1000research.24651.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33123349</td>\n",
       "      <td>cc-by</td>\n",
       "      <td>Background: The ability to protect workers and...</td>\n",
       "      <td>2020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.12688/f1000research.24651.1...</td>\n",
       "      <td>221147881.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>In vitro efficacy of a copper iodine complex P...</td>\n",
       "      <td>[101, 1107, 191, 2875, 2180, 23891, 1104, 170,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32350226</th>\n",
       "      <td>jlcgjjal</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>An assessment of real-time RT-PCR kits for SAR...</td>\n",
       "      <td>10.7883/yoken.jjid.2020.108</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32350226</td>\n",
       "      <td>unk</td>\n",
       "      <td>The COVID-19 outbreak caused by SARS-CoV-2 in ...</td>\n",
       "      <td>2020-04-30</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.7883/yoken.jjid.2020.108; h...</td>\n",
       "      <td>217593691.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>An assessment of real-time RT-PCR kits for SAR...</td>\n",
       "      <td>[101, 1126, 8670, 1104, 1842, 118, 1159, 187, ...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33530819</th>\n",
       "      <td>3ogk7qis</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Peripheral blood microRNAs and the COVID-19 pa...</td>\n",
       "      <td>10.1080/15476286.2021.1885188</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33530819</td>\n",
       "      <td>unk</td>\n",
       "      <td>The COVID-19 emergency pandemic resulting from...</td>\n",
       "      <td>2021-02-02</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1080/15476286.2021.1885188;...</td>\n",
       "      <td>231788938.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Peripheral blood microRNAs and the COVID-19 pa...</td>\n",
       "      <td>[101, 17963, 1892, 17599, 11782, 1116, 1105, 1...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33646036</th>\n",
       "      <td>k0zw8nvd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>COVID-19 vaccines: what happened to evidence-b...</td>\n",
       "      <td>10.12968/hmed.2021.0047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33646036</td>\n",
       "      <td>unk</td>\n",
       "      <td>The UK government recently decided to extend t...</td>\n",
       "      <td>2021-02-02</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.12968/hmed.2021.0047; https...</td>\n",
       "      <td>232078909.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>COVID-19 vaccines: what happened to evidence-b...</td>\n",
       "      <td>[101, 1884, 18312, 118, 1627, 20034, 1116, 131...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32413164</th>\n",
       "      <td>if4nuhgz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Delivering psychotherapy by video conference i...</td>\n",
       "      <td>10.1111/jpm.12659</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32413164</td>\n",
       "      <td>unk</td>\n",
       "      <td>In New Zealand we have been far less affected ...</td>\n",
       "      <td>2020-05-15</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/jpm.12659; https://www...</td>\n",
       "      <td>218659039.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Delivering psychotherapy by video conference i...</td>\n",
       "      <td>[101, 13232, 15604, 21155, 20939, 1118, 1888, ...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33183019</th>\n",
       "      <td>qn2rlzb4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Complete and enduring response in an elderly p...</td>\n",
       "      <td>10.21037/apm-20-1485</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33183019</td>\n",
       "      <td>cc-by-nc-nd</td>\n",
       "      <td>Combined immune checkpoint inhibitor with plat...</td>\n",
       "      <td>2020-10-27</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.21037/apm-20-1485; https://...</td>\n",
       "      <td>226855189.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Complete and enduring response in an elderly p...</td>\n",
       "      <td>[101, 2335, 1105, 21585, 2593, 1107, 1126, 980...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32549917</th>\n",
       "      <td>jgoinon3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Sleep disorders in migrants and refugees: a sy...</td>\n",
       "      <td>10.1007/s13167-020-00205-2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32549917</td>\n",
       "      <td>unk</td>\n",
       "      <td>Background Sleep disorders are very common in ...</td>\n",
       "      <td>2020-06-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1007/s13167-020-00205-2; ht...</td>\n",
       "      <td>218624800.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Sleep disorders in migrants and refugees: a sy...</td>\n",
       "      <td>[101, 2946, 11759, 1107, 18677, 1105, 8940, 13...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29938891</th>\n",
       "      <td>7pl5kjyl</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Measuring beverage consumption in US children ...</td>\n",
       "      <td>10.1111/obr.12692</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29938891</td>\n",
       "      <td>unk</td>\n",
       "      <td>Beverage consumption habits are associated wit...</td>\n",
       "      <td>2018</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1111/obr.12692; https://www...</td>\n",
       "      <td>49418389.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Measuring beverage consumption in US children ...</td>\n",
       "      <td>[101, 10099, 24802, 8160, 1107, 1366, 1482, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33406963</th>\n",
       "      <td>pro1yt7f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Brazilian Older People Hospitalized by COVID-1...</td>\n",
       "      <td>10.1177/0733464820983976</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33406963</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVE Analyzing the survival of older peop...</td>\n",
       "      <td>2021-01-06</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1177/0733464820983976; http...</td>\n",
       "      <td>230817520.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Brazilian Older People Hospitalized by COVID-1...</td>\n",
       "      <td>[101, 12418, 5303, 15647, 2214, 1234, 2704, 22...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33236629</th>\n",
       "      <td>sswn2f1d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Specificity Enhancement of Deoxyribonucleic Ac...</td>\n",
       "      <td>10.1021/acs.analchem.0c03223</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33236629</td>\n",
       "      <td>unk</td>\n",
       "      <td>Specificity of DNA polymerization plays a crit...</td>\n",
       "      <td>2020-11-25</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1021/acs.analchem.0c03223; ...</td>\n",
       "      <td>227167582.0</td>\n",
       "      <td>excluded</td>\n",
       "      <td>Specificity Enhancement of Deoxyribonucleic Ac...</td>\n",
       "      <td>[101, 2747, 1785, 11778, 1880, 1104, 1260, 106...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27490600</th>\n",
       "      <td>a15cl3ur</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Near Infrared Spectroscopy as a Hemodynamic Mo...</td>\n",
       "      <td>10.1097/pcc.0000000000000780</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27490600</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBJECTIVES The objectives of this review are t...</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1097/pcc.0000000000000780; ...</td>\n",
       "      <td>40483525.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Near Infrared Spectroscopy as a Hemodynamic Mo...</td>\n",
       "      <td>[101, 1485, 18688, 188, 26426, 5864, 20739, 11...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31991422</th>\n",
       "      <td>88nttxtm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Epidural-Related Fever and Maternal and Neonat...</td>\n",
       "      <td>10.1159/000504805</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31991422</td>\n",
       "      <td>cc-by-nc-nd</td>\n",
       "      <td>BACKGROUND While epidural analgesia (EA) is as...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1159/000504805; https://www...</td>\n",
       "      <td>210945568.0</td>\n",
       "      <td>systematic-review</td>\n",
       "      <td>Epidural-Related Fever and Maternal and Neonat...</td>\n",
       "      <td>[101, 174, 25786, 12602, 118, 2272, 10880, 110...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32409432</th>\n",
       "      <td>mz7nuskv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Home monitoring for COVID-19.</td>\n",
       "      <td>10.3949/ccjm.87a.ccc028</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32409432</td>\n",
       "      <td>unk</td>\n",
       "      <td>Cleveland Clinic recognized the importance of ...</td>\n",
       "      <td>2020-05-14</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.3949/ccjm.87a.ccc028; https...</td>\n",
       "      <td>218649040.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Home monitoring for COVID-19. Cleveland Clinic...</td>\n",
       "      <td>[101, 1313, 9437, 1111, 1884, 18312, 118, 1627...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19477503</th>\n",
       "      <td>37zrbye6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Effectiveness of thigh-length graduated compre...</td>\n",
       "      <td>10.1016/s0140-6736(09)60941-7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19477503</td>\n",
       "      <td>unk</td>\n",
       "      <td>BACKGROUND Deep vein thrombosis (DVT) and pulm...</td>\n",
       "      <td>2009</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1016/s0140-6736(09)60941-7;...</td>\n",
       "      <td>11718220.0</td>\n",
       "      <td>primary-rct</td>\n",
       "      <td>Effectiveness of thigh-length graduated compre...</td>\n",
       "      <td>[101, 12949, 1104, 9973, 118, 2251, 3024, 1492...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32816516</th>\n",
       "      <td>9uwlkncw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>What the world could learn from the Haitian re...</td>\n",
       "      <td>10.1037/tra0000903</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32816516</td>\n",
       "      <td>unk</td>\n",
       "      <td>In the 1st trimester of 2020, there were mixed...</td>\n",
       "      <td>2020-09-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1037/tra0000903; https://ww...</td>\n",
       "      <td>221222419.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>What the world could learn from the Haitian re...</td>\n",
       "      <td>[101, 1184, 1103, 1362, 1180, 3858, 1121, 1103...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32802446</th>\n",
       "      <td>rl93ipgj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Investigation of a family cluster outbreak of ...</td>\n",
       "      <td>10.21037/jtd-20-955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32802446</td>\n",
       "      <td>cc-by-nc-nd</td>\n",
       "      <td>Background To investigate whether asymptomatic...</td>\n",
       "      <td>2020-07-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.21037/jtd-20-955; https://w...</td>\n",
       "      <td>225756814.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Investigation of a family cluster outbreak of ...</td>\n",
       "      <td>[101, 4449, 1104, 170, 1266, 10005, 8010, 1104...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33351483</th>\n",
       "      <td>m8l7xyhc</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Yield of Adding chest CT to Abdominal CT to De...</td>\n",
       "      <td>10.1097/sla.0000000000004678</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33351483</td>\n",
       "      <td>unk</td>\n",
       "      <td>OBTECTIVE To determine the incremental yield o...</td>\n",
       "      <td>2020-12-18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1097/sla.0000000000004678; ...</td>\n",
       "      <td>229696117.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Yield of Adding chest CT to Abdominal CT to De...</td>\n",
       "      <td>[101, 10972, 1104, 5321, 2229, 172, 1204, 1106...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33044493</th>\n",
       "      <td>irgnrr0j</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Outcomes of Neonates Born to Mothers With Seve...</td>\n",
       "      <td>10.1001/jamapediatrics.2020.4298</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33044493</td>\n",
       "      <td>unk</td>\n",
       "      <td>Importance Limited data on vertical and perina...</td>\n",
       "      <td>2020-10-12</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1001/jamapediatrics.2020.42...</td>\n",
       "      <td>222302311.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Outcomes of Neonates Born to Mothers With Seve...</td>\n",
       "      <td>[101, 13950, 1104, 24762, 5430, 1255, 1106, 12...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32443114</th>\n",
       "      <td>s26ui4kz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>Where Are the Masks?</td>\n",
       "      <td>10.1097/01.naj.0000668700.23009.1e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32443114</td>\n",
       "      <td>unk</td>\n",
       "      <td>Shortages of vital PPE during the COVID-19 pan...</td>\n",
       "      <td>2020-06-01</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.1097/01.naj.0000668700.2300...</td>\n",
       "      <td>218858413.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>Where Are the Masks? Shortages of vital PPE du...</td>\n",
       "      <td>[101, 1187, 1132, 1103, 17944, 136, 25630, 110...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33757196</th>\n",
       "      <td>e8f5e234</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Medline</td>\n",
       "      <td>A deep learning-based medication behavior moni...</td>\n",
       "      <td>10.3934/mbe.2021078</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33757196</td>\n",
       "      <td>unk</td>\n",
       "      <td>The internet of things (IoT) and deep learning...</td>\n",
       "      <td>2021-01-28</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://doi.org/10.3934/mbe.2021078; https://w...</td>\n",
       "      <td>232340398.0</td>\n",
       "      <td>primary-not-rct</td>\n",
       "      <td>A deep learning-based medication behavior moni...</td>\n",
       "      <td>[101, 170, 1996, 3776, 118, 1359, 15683, 4658,...</td>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18854 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           cord_uid  sha source_x  \\\n",
       "pubmed_id                           \n",
       "32572175   jq2em42q  NaN  Medline   \n",
       "33079612   zcxwnvy2  NaN  Medline   \n",
       "33565705   izqvn6nr  NaN  Medline   \n",
       "33479162   r17yloy3  NaN  Medline   \n",
       "33705595   20p9lr62  NaN  Medline   \n",
       "33771818   9asnam5x  NaN  Medline   \n",
       "32890322   y4p2ogrz  NaN  Medline   \n",
       "33381888   9zi08xl7  NaN  Medline   \n",
       "32628408   c1xd4mtz  NaN  Medline   \n",
       "33179469   ws1xhm52  NaN  Medline   \n",
       "33305712   ewhqdywx  NaN  Medline   \n",
       "32920815   0wu2pl1r  NaN  Medline   \n",
       "32657602   a980qiu9  NaN  Medline   \n",
       "32601125   s7u0ct2l  NaN  Medline   \n",
       "33620014   203ib111  NaN  Medline   \n",
       "33533477   97s2ieqi  NaN  Medline   \n",
       "33290467   4yalw4qa  NaN  Medline   \n",
       "33645461   jc38sall  NaN  Medline   \n",
       "33004394   qvil8upe  NaN  Medline   \n",
       "33739446   iudemqdy  NaN  Medline   \n",
       "33557718   6paiox0q  NaN  Medline   \n",
       "32459567   wyln1jpr  NaN  Medline   \n",
       "33615885   fy46t6hm  NaN  Medline   \n",
       "33315291   upkafkkf  NaN  Medline   \n",
       "32866992   ni5oyytr  NaN  Medline   \n",
       "28958212   pip5ybqd  NaN  Medline   \n",
       "33647224   9hajj5cz  NaN  Medline   \n",
       "33770334   7r5cz2qc  NaN  Medline   \n",
       "33569794   mqvho63h  NaN  Medline   \n",
       "33368849   kyq4ek3s  NaN  Medline   \n",
       "...             ...  ...      ...   \n",
       "33283686   hickh15h  NaN  Medline   \n",
       "33730407   4siasmyn  NaN  Medline   \n",
       "28840782   gi3oh9o8  NaN  Medline   \n",
       "32419599   i9nr0zdi  NaN  Medline   \n",
       "33526471   7douonww  NaN  Medline   \n",
       "31920368   kl2766oe  NaN  Medline   \n",
       "33141520   aafmh4h9  NaN  Medline   \n",
       "33085086   7gceby4t  NaN  Medline   \n",
       "33744954   tqil35of  NaN  Medline   \n",
       "32571147   8mcrvtp2  NaN  Medline   \n",
       "33123349   c776npf0  NaN  Medline   \n",
       "32350226   jlcgjjal  NaN  Medline   \n",
       "33530819   3ogk7qis  NaN  Medline   \n",
       "33646036   k0zw8nvd  NaN  Medline   \n",
       "32413164   if4nuhgz  NaN  Medline   \n",
       "33183019   qn2rlzb4  NaN  Medline   \n",
       "32549917   jgoinon3  NaN  Medline   \n",
       "29938891   7pl5kjyl  NaN  Medline   \n",
       "33406963   pro1yt7f  NaN  Medline   \n",
       "33236629   sswn2f1d  NaN  Medline   \n",
       "27490600   a15cl3ur  NaN  Medline   \n",
       "31991422   88nttxtm  NaN  Medline   \n",
       "32409432   mz7nuskv  NaN  Medline   \n",
       "19477503   37zrbye6  NaN  Medline   \n",
       "32816516   9uwlkncw  NaN  Medline   \n",
       "32802446   rl93ipgj  NaN  Medline   \n",
       "33351483   m8l7xyhc  NaN  Medline   \n",
       "33044493   irgnrr0j  NaN  Medline   \n",
       "32443114   s26ui4kz  NaN  Medline   \n",
       "33757196   e8f5e234  NaN  Medline   \n",
       "\n",
       "                                                       title  \\\n",
       "pubmed_id                                                      \n",
       "32572175   The end of social confinement and COVID-19 re-...   \n",
       "33079612      The Missing Link in the Covid-19 Vaccine Race.   \n",
       "33565705   Donor To Recipient Transmission Of SARS-CoV-2 ...   \n",
       "33479162   Neonates Born to Mothers With COVID-19: Data F...   \n",
       "33705595   New targets for drug design: Importance of nsp...   \n",
       "33771818   Establishing a COVID-19 treatment centre in Is...   \n",
       "32890322   Guidelines for Ultrasound in the Radiology Dep...   \n",
       "33381888   Antiphospholipid antibodies and thrombosis in ...   \n",
       "32628408   Nonoperating room anesthesia for patients with...   \n",
       "33179469   Surgical management protocol during the COVID-...   \n",
       "33305712            COVID-19 Severity: Lung-Heart Interplay.   \n",
       "32920815   Novel coronavirus(Covid19), Its renal manifest...   \n",
       "32657602   The COVID-19 Outbreak in Iran; The First Patie...   \n",
       "32601125   Potential role of endothelial cell surface ect...   \n",
       "33620014   Convalescent Plasma and Hyperimmune Globulin T...   \n",
       "33533477   Nitric oxide and Covid-19: dose, timing and ho...   \n",
       "33290467    Introductory lecture: air quality in megacities.   \n",
       "33645461   Pneumocystis jirovecii pneumonia in an immunoc...   \n",
       "33004394   Nutrition screening and assessment tools for p...   \n",
       "33739446   Left-behind children's social adjustment and r...   \n",
       "33557718   Social work leadership in a medical school: A ...   \n",
       "32459567   Use of systemic therapies for psoriasis in the...   \n",
       "33615885   Do recipients of the first dose of the Pfizer/...   \n",
       "33315291   Does inflammation link stress to poor COVID-19...   \n",
       "32866992   Paucity of COVID-19 Dermatology Literature fro...   \n",
       "28958212   Effectiveness of remote feedback on physical a...   \n",
       "33647224   Results of preoperative screening for COVID-19...   \n",
       "33770334   The Impact of COVID-19 on Access to HIV Preven...   \n",
       "33569794   New thing, The Impact of the Emergence of COVI...   \n",
       "33368849   Perceived stress and social support in a large...   \n",
       "...                                                      ...   \n",
       "33283686           Effect of coronaphobia on smoking habits.   \n",
       "33730407   A simple, safe and sensitive method for SARS-C...   \n",
       "28840782   Comparison of three blood transfusion guidelin...   \n",
       "32419599   Are there positive lessons for Italy's NHS res...   \n",
       "33526471   A Novel Soluble ACE2 Variant with Prolonged Du...   \n",
       "31920368   Nonpharmacological Interventions Addressing Pa...   \n",
       "33141520   Proteomic Biomarkers to Guide Stratification f...   \n",
       "33085086   Increased occurrence of hand eczema in young c...   \n",
       "33744954   Anti-SARS-CoV2 antibody responses in serum and...   \n",
       "32571147   The Difficult Airway and Aerosol-Generating Pr...   \n",
       "33123349   In vitro efficacy of a copper iodine complex P...   \n",
       "32350226   An assessment of real-time RT-PCR kits for SAR...   \n",
       "33530819   Peripheral blood microRNAs and the COVID-19 pa...   \n",
       "33646036   COVID-19 vaccines: what happened to evidence-b...   \n",
       "32413164   Delivering psychotherapy by video conference i...   \n",
       "33183019   Complete and enduring response in an elderly p...   \n",
       "32549917   Sleep disorders in migrants and refugees: a sy...   \n",
       "29938891   Measuring beverage consumption in US children ...   \n",
       "33406963   Brazilian Older People Hospitalized by COVID-1...   \n",
       "33236629   Specificity Enhancement of Deoxyribonucleic Ac...   \n",
       "27490600   Near Infrared Spectroscopy as a Hemodynamic Mo...   \n",
       "31991422   Epidural-Related Fever and Maternal and Neonat...   \n",
       "32409432                       Home monitoring for COVID-19.   \n",
       "19477503   Effectiveness of thigh-length graduated compre...   \n",
       "32816516   What the world could learn from the Haitian re...   \n",
       "32802446   Investigation of a family cluster outbreak of ...   \n",
       "33351483   Yield of Adding chest CT to Abdominal CT to De...   \n",
       "33044493   Outcomes of Neonates Born to Mothers With Seve...   \n",
       "32443114                                Where Are the Masks?   \n",
       "33757196   A deep learning-based medication behavior moni...   \n",
       "\n",
       "                                          doi  pmcid  pubmed_id      license  \\\n",
       "pubmed_id                                                                      \n",
       "32572175            10.1038/s41562-020-0908-8    NaN   32572175          unk   \n",
       "33079612        10.1080/21645515.2020.1831859    NaN   33079612          unk   \n",
       "33565705                    10.1111/ajt.16532    NaN   33565705          unk   \n",
       "33479162             10.1542/peds.2020-015065    NaN   33479162          unk   \n",
       "33705595                   10.1111/febs.15815    NaN   33705595          unk   \n",
       "33771818          10.1136/emermed-2020-209639    NaN   33771818          unk   \n",
       "32890322         10.1097/ruq.0000000000000526    NaN   32890322          unk   \n",
       "33381888                    10.1002/art.41634    NaN   33381888          unk   \n",
       "32628408         10.1097/aco.0000000000000890    NaN   32628408          unk   \n",
       "33179469       10.23736/s0026-4733.20.08632-0    NaN   33179469          unk   \n",
       "33305712    10.2174/1573403x16999201210200614    NaN   33305712          unk   \n",
       "32920815               10.22037/uj.v16i7.6431    NaN   32920815          unk   \n",
       "32657602                 10.34172/aim.2020.48    NaN   32657602        cc-by   \n",
       "32601125            10.7861/clinmed.2020-0252    NaN   32601125          unk   \n",
       "33620014        10.1080/1744666x.2021.1894927    NaN   33620014          unk   \n",
       "33533477                    10.1111/aas.13788    NaN   33533477          unk   \n",
       "33290467                   10.1039/d0fd00123f    NaN   33290467     cc-by-nc   \n",
       "33645461        10.1080/23744235.2021.1890331    NaN   33645461          unk   \n",
       "33004394          10.1136/bmjopen-2020-037844    NaN   33004394     cc-by-nc   \n",
       "33739446                   10.1002/ijop.12754    NaN   33739446          unk   \n",
       "33557718        10.1080/00981389.2021.1885567    NaN   33557718          unk   \n",
       "32459567        10.1080/09546634.2020.1775774    NaN   32459567          unk   \n",
       "33615885             10.1177/0025817221990520    NaN   33615885          unk   \n",
       "33315291                     10.1002/smi.3017    NaN   33315291          unk   \n",
       "32866992                    10.1111/bjd.19505    NaN   32866992          unk   \n",
       "28958212             10.1177/1357633x17733772    NaN   28958212          unk   \n",
       "33647224        10.1080/21548331.2021.1898158    NaN   33647224          unk   \n",
       "33770334           10.1007/s10900-021-00979-0    NaN   33770334          unk   \n",
       "33569794                      10.1002/pd.5902    NaN   33569794          unk   \n",
       "33368849                    10.1111/ene.14697    NaN   33368849          unk   \n",
       "...                                       ...    ...        ...          ...   \n",
       "33283686        10.1080/10550887.2020.1849950    NaN   33283686          unk   \n",
       "33730407                    10.1111/apm.13123    NaN   33730407          unk   \n",
       "28840782             10.1177/1098612x17727233    NaN   28840782          unk   \n",
       "32419599             10.1177/0025817220923698    NaN   32419599          unk   \n",
       "33526471               10.1681/asn.2020101537    NaN   33526471          unk   \n",
       "31920368                  10.2147/jpr.s216807    NaN   31920368     cc-by-nc   \n",
       "33141520               10.1002/pmic.202000229    NaN   33141520        cc-by   \n",
       "33085086                    10.1111/cod.13727    NaN   33085086          unk   \n",
       "33744954               10.1093/infdis/jiab153    NaN   33744954          unk   \n",
       "32571147             10.1177/0194599820936615    NaN   32571147          unk   \n",
       "33123349       10.12688/f1000research.24651.1    NaN   33123349        cc-by   \n",
       "32350226          10.7883/yoken.jjid.2020.108    NaN   32350226          unk   \n",
       "33530819        10.1080/15476286.2021.1885188    NaN   33530819          unk   \n",
       "33646036              10.12968/hmed.2021.0047    NaN   33646036          unk   \n",
       "32413164                    10.1111/jpm.12659    NaN   32413164          unk   \n",
       "33183019                 10.21037/apm-20-1485    NaN   33183019  cc-by-nc-nd   \n",
       "32549917           10.1007/s13167-020-00205-2    NaN   32549917          unk   \n",
       "29938891                    10.1111/obr.12692    NaN   29938891          unk   \n",
       "33406963             10.1177/0733464820983976    NaN   33406963          unk   \n",
       "33236629         10.1021/acs.analchem.0c03223    NaN   33236629          unk   \n",
       "27490600         10.1097/pcc.0000000000000780    NaN   27490600          unk   \n",
       "31991422                    10.1159/000504805    NaN   31991422  cc-by-nc-nd   \n",
       "32409432              10.3949/ccjm.87a.ccc028    NaN   32409432          unk   \n",
       "19477503        10.1016/s0140-6736(09)60941-7    NaN   19477503          unk   \n",
       "32816516                   10.1037/tra0000903    NaN   32816516          unk   \n",
       "32802446                  10.21037/jtd-20-955    NaN   32802446  cc-by-nc-nd   \n",
       "33351483         10.1097/sla.0000000000004678    NaN   33351483          unk   \n",
       "33044493     10.1001/jamapediatrics.2020.4298    NaN   33044493          unk   \n",
       "32443114   10.1097/01.naj.0000668700.23009.1e    NaN   32443114          unk   \n",
       "33757196                  10.3934/mbe.2021078    NaN   33757196          unk   \n",
       "\n",
       "                                                    abstract publish_time  \\\n",
       "pubmed_id                                                                   \n",
       "32572175   The lack of effective pharmaceutical intervent...   2020-06-22   \n",
       "33079612   Operation Warp Speed and global vaccine resear...   2020-10-20   \n",
       "33565705   We describe a case of proven transmission of S...   2021-02-10   \n",
       "33479162   OBJECTIVES To describe neonatal and maternal c...   2021-01-21   \n",
       "33705595   SARS-CoV-2 virus has triggered a global pandem...   2021-03-11   \n",
       "33771818   Anticipating the need for a COVID-19 treatment...   2021-03-26   \n",
       "32890322   The coronavirus disease 2019 is caused by the ...   2020-09-01   \n",
       "33381888   We read with great interest the study by Berti...   2020-12-31   \n",
       "32628408   PURPOSE OF REVIEW To provide aids to deal with...   2020-08-01   \n",
       "33179469   BACKGROUND In the surgical scenario, the sever...   2020-11-11   \n",
       "33305712   In December 2019, a novel COVID-19 caused by S...   2020-12-10   \n",
       "32920815                                        No abstract.   2020-09-13   \n",
       "32657602   The novel coronavirus, formerly named as 2019 ...   2020-07-01   \n",
       "32601125   The novel coronavirus infectious disease (COVI...   2020-06-29   \n",
       "33620014   INTRODUCTION Severe acute respiratory syndrome...   2021-02-23   \n",
       "33533477   Nitric oxide (NO) for inhalation has been a cl...   2021-02-03   \n",
       "33290467   Urbanization is an ongoing global phenomenon a...   2020-12-08   \n",
       "33645461   BACKGROUND Several cases of invasive fungal di...   2021-03-01   \n",
       "33004394   INTRODUCTION Nutritional challenges are common...   2020-10-01   \n",
       "33739446   Using data collected from two provinces in Chi...   2021-03-19   \n",
       "33557718   The COVID-19 pandemic has exposed the systemic...   2021-02-09   \n",
       "32459567   BACKGROUND In late 2019 a viral pneumonia bega...   2020-05-27   \n",
       "33615885   The Covid-19 pandemic in the UK has been great...   2021-02-22   \n",
       "33315291   Coronavirus disease 2019 (COVID-19) continues ...   2020-12-14   \n",
       "32866992   Since emerging in December 2019, COVID-19 has ...   2020-08-31   \n",
       "28958212   OBJECTIVES The objective of this systematic re...         2019   \n",
       "33647224   OBJECTIVES Many hospitals recently instituted ...   2021-03-01   \n",
       "33770334   The COVID-19 pandemic has necessitated restric...   2021-03-26   \n",
       "33569794   OBJECTIVE We conducted a study to examine the ...   2021-02-10   \n",
       "33368849   BACKGROUND The COVID-19 emergency may cause po...   2020-12-23   \n",
       "...                                                      ...          ...   \n",
       "33283686   INTRODUCTION AND OBJECTIVE Coronavirus disease...   2020-12-07   \n",
       "33730407   The SARS-CoV-2 pandemic has created an urgent ...   2021-03-17   \n",
       "28840782   Objectives The increased demand for animal blo...         2018   \n",
       "32419599   The authors evaluate the importance of prevent...   2020-05-18   \n",
       "33526471   BACKGROUND There is an urgent need for approac...   2021-02-01   \n",
       "31920368   Purpose Children and adolescents with primary ...         2019   \n",
       "33141520   Proteins and peptides serve as biomarkers in t...   2020-11-03   \n",
       "33085086   BACKGROUND During the first wave of the COVID-...   2020-10-21   \n",
       "33744954   Antibody responses to SARS-CoV-2 in serum and ...   2021-03-21   \n",
       "32571147   The impact of the COVID-19 pandemic on otolary...   2020-06-23   \n",
       "33123349   Background: The ability to protect workers and...         2020   \n",
       "32350226   The COVID-19 outbreak caused by SARS-CoV-2 in ...   2020-04-30   \n",
       "33530819   The COVID-19 emergency pandemic resulting from...   2021-02-02   \n",
       "33646036   The UK government recently decided to extend t...   2021-02-02   \n",
       "32413164   In New Zealand we have been far less affected ...   2020-05-15   \n",
       "33183019   Combined immune checkpoint inhibitor with plat...   2020-10-27   \n",
       "32549917   Background Sleep disorders are very common in ...   2020-06-01   \n",
       "29938891   Beverage consumption habits are associated wit...         2018   \n",
       "33406963   OBJECTIVE Analyzing the survival of older peop...   2021-01-06   \n",
       "33236629   Specificity of DNA polymerization plays a crit...   2020-11-25   \n",
       "27490600   OBJECTIVES The objectives of this review are t...         2016   \n",
       "31991422   BACKGROUND While epidural analgesia (EA) is as...   2020-01-28   \n",
       "32409432   Cleveland Clinic recognized the importance of ...   2020-05-14   \n",
       "19477503   BACKGROUND Deep vein thrombosis (DVT) and pulm...         2009   \n",
       "32816516   In the 1st trimester of 2020, there were mixed...   2020-09-01   \n",
       "32802446   Background To investigate whether asymptomatic...   2020-07-01   \n",
       "33351483   OBTECTIVE To determine the incremental yield o...   2020-12-18   \n",
       "33044493   Importance Limited data on vertical and perina...   2020-10-12   \n",
       "32443114   Shortages of vital PPE during the COVID-19 pan...   2020-06-01   \n",
       "33757196   The internet of things (IoT) and deep learning...   2021-01-28   \n",
       "\n",
       "           ... who_covidence_id arxiv_id  pdf_json_files  pmc_json_files  \\\n",
       "pubmed_id  ...                                                             \n",
       "32572175   ...              NaN      NaN             NaN             NaN   \n",
       "33079612   ...              NaN      NaN             NaN             NaN   \n",
       "33565705   ...              NaN      NaN             NaN             NaN   \n",
       "33479162   ...              NaN      NaN             NaN             NaN   \n",
       "33705595   ...              NaN      NaN             NaN             NaN   \n",
       "33771818   ...              NaN      NaN             NaN             NaN   \n",
       "32890322   ...              NaN      NaN             NaN             NaN   \n",
       "33381888   ...              NaN      NaN             NaN             NaN   \n",
       "32628408   ...              NaN      NaN             NaN             NaN   \n",
       "33179469   ...              NaN      NaN             NaN             NaN   \n",
       "33305712   ...              NaN      NaN             NaN             NaN   \n",
       "32920815   ...              NaN      NaN             NaN             NaN   \n",
       "32657602   ...              NaN      NaN             NaN             NaN   \n",
       "32601125   ...              NaN      NaN             NaN             NaN   \n",
       "33620014   ...              NaN      NaN             NaN             NaN   \n",
       "33533477   ...              NaN      NaN             NaN             NaN   \n",
       "33290467   ...              NaN      NaN             NaN             NaN   \n",
       "33645461   ...              NaN      NaN             NaN             NaN   \n",
       "33004394   ...              NaN      NaN             NaN             NaN   \n",
       "33739446   ...              NaN      NaN             NaN             NaN   \n",
       "33557718   ...              NaN      NaN             NaN             NaN   \n",
       "32459567   ...              NaN      NaN             NaN             NaN   \n",
       "33615885   ...              NaN      NaN             NaN             NaN   \n",
       "33315291   ...              NaN      NaN             NaN             NaN   \n",
       "32866992   ...              NaN      NaN             NaN             NaN   \n",
       "28958212   ...              NaN      NaN             NaN             NaN   \n",
       "33647224   ...              NaN      NaN             NaN             NaN   \n",
       "33770334   ...              NaN      NaN             NaN             NaN   \n",
       "33569794   ...              NaN      NaN             NaN             NaN   \n",
       "33368849   ...              NaN      NaN             NaN             NaN   \n",
       "...        ...              ...      ...             ...             ...   \n",
       "33283686   ...              NaN      NaN             NaN             NaN   \n",
       "33730407   ...              NaN      NaN             NaN             NaN   \n",
       "28840782   ...              NaN      NaN             NaN             NaN   \n",
       "32419599   ...              NaN      NaN             NaN             NaN   \n",
       "33526471   ...              NaN      NaN             NaN             NaN   \n",
       "31920368   ...              NaN      NaN             NaN             NaN   \n",
       "33141520   ...              NaN      NaN             NaN             NaN   \n",
       "33085086   ...              NaN      NaN             NaN             NaN   \n",
       "33744954   ...              NaN      NaN             NaN             NaN   \n",
       "32571147   ...              NaN      NaN             NaN             NaN   \n",
       "33123349   ...              NaN      NaN             NaN             NaN   \n",
       "32350226   ...              NaN      NaN             NaN             NaN   \n",
       "33530819   ...              NaN      NaN             NaN             NaN   \n",
       "33646036   ...              NaN      NaN             NaN             NaN   \n",
       "32413164   ...              NaN      NaN             NaN             NaN   \n",
       "33183019   ...              NaN      NaN             NaN             NaN   \n",
       "32549917   ...              NaN      NaN             NaN             NaN   \n",
       "29938891   ...              NaN      NaN             NaN             NaN   \n",
       "33406963   ...              NaN      NaN             NaN             NaN   \n",
       "33236629   ...              NaN      NaN             NaN             NaN   \n",
       "27490600   ...              NaN      NaN             NaN             NaN   \n",
       "31991422   ...              NaN      NaN             NaN             NaN   \n",
       "32409432   ...              NaN      NaN             NaN             NaN   \n",
       "19477503   ...              NaN      NaN             NaN             NaN   \n",
       "32816516   ...              NaN      NaN             NaN             NaN   \n",
       "32802446   ...              NaN      NaN             NaN             NaN   \n",
       "33351483   ...              NaN      NaN             NaN             NaN   \n",
       "33044493   ...              NaN      NaN             NaN             NaN   \n",
       "32443114   ...              NaN      NaN             NaN             NaN   \n",
       "33757196   ...              NaN      NaN             NaN             NaN   \n",
       "\n",
       "                                                         url        s2_id  \\\n",
       "pubmed_id                                                                   \n",
       "32572175   https://doi.org/10.1038/s41562-020-0908-8; htt...  219958669.0   \n",
       "33079612   https://doi.org/10.1080/21645515.2020.1831859;...  224826061.0   \n",
       "33565705   https://doi.org/10.1111/ajt.16532; https://www...  231872023.0   \n",
       "33479162   https://doi.org/10.1542/peds.2020-015065; http...  231676588.0   \n",
       "33705595   https://doi.org/10.1111/febs.15815; https://ww...  232208657.0   \n",
       "33771818   https://doi.org/10.1136/emermed-2020-209639; h...  232369853.0   \n",
       "32890322   https://doi.org/10.1097/ruq.0000000000000526; ...  221511545.0   \n",
       "33381888   https://doi.org/10.1002/art.41634; https://www...  229928885.0   \n",
       "32628408   https://doi.org/10.1097/aco.0000000000000890; ...  220388324.0   \n",
       "33179469   https://doi.org/10.23736/s0026-4733.20.08632-0...  226310156.0   \n",
       "33305712   https://doi.org/10.2174/1573403x16999201210200...  228100886.0   \n",
       "32920815   https://doi.org/10.22037/uj.v16i7.6431; https:...  221674559.0   \n",
       "32657602   https://doi.org/10.34172/aim.2020.48; https://...  220500434.0   \n",
       "32601125   https://doi.org/10.7861/clinmed.2020-0252; htt...  220268336.0   \n",
       "33620014   https://doi.org/10.1080/1744666x.2021.1894927;...  232019282.0   \n",
       "33533477   https://doi.org/10.1111/aas.13788; https://www...  231789608.0   \n",
       "33290467   https://doi.org/10.1039/d0fd00123f; https://ww...  228081191.0   \n",
       "33645461   https://doi.org/10.1080/23744235.2021.1890331;...  232079882.0   \n",
       "33004394   https://doi.org/10.1136/bmjopen-2020-037844; h...  222145118.0   \n",
       "33739446   https://doi.org/10.1002/ijop.12754; https://ww...  232305011.0   \n",
       "33557718   https://doi.org/10.1080/00981389.2021.1885567;...  231864146.0   \n",
       "32459567   https://doi.org/10.1080/09546634.2020.1775774;...  218975132.0   \n",
       "33615885   https://doi.org/10.1177/0025817221990520; http...  231988376.0   \n",
       "33315291   https://doi.org/10.1002/smi.3017; https://www....  229175021.0   \n",
       "32866992   https://doi.org/10.1111/bjd.19505; https://www...  221405192.0   \n",
       "28958212   https://doi.org/10.1177/1357633x17733772; http...   35565293.0   \n",
       "33647224   https://doi.org/10.1080/21548331.2021.1898158;...  232091699.0   \n",
       "33770334   https://doi.org/10.1007/s10900-021-00979-0; ht...  232371065.0   \n",
       "33569794   https://doi.org/10.1002/pd.5902; https://www.n...  231882629.0   \n",
       "33368849   https://doi.org/10.1111/ene.14697; https://www...  229691636.0   \n",
       "...                                                      ...          ...   \n",
       "33283686   https://doi.org/10.1080/10550887.2020.1849950;...  227524516.0   \n",
       "33730407   https://doi.org/10.1111/apm.13123; https://www...  232300402.0   \n",
       "28840782   https://doi.org/10.1177/1098612x17727233; http...   40081141.0   \n",
       "32419599   https://doi.org/10.1177/0025817220923698; http...  218677522.0   \n",
       "33526471   https://doi.org/10.1681/asn.2020101537; https:...  231760153.0   \n",
       "31920368   https://doi.org/10.2147/jpr.s216807; https://w...  210118378.0   \n",
       "33141520   https://doi.org/10.1002/pmic.202000229; https:...  226244341.0   \n",
       "33085086   https://doi.org/10.1111/cod.13727; https://www...  224829028.0   \n",
       "33744954   https://doi.org/10.1093/infdis/jiab153; https:...  232310704.0   \n",
       "32571147   https://doi.org/10.1177/0194599820936615; http...  219991330.0   \n",
       "33123349   https://doi.org/10.12688/f1000research.24651.1...  221147881.0   \n",
       "32350226   https://doi.org/10.7883/yoken.jjid.2020.108; h...  217593691.0   \n",
       "33530819   https://doi.org/10.1080/15476286.2021.1885188;...  231788938.0   \n",
       "33646036   https://doi.org/10.12968/hmed.2021.0047; https...  232078909.0   \n",
       "32413164   https://doi.org/10.1111/jpm.12659; https://www...  218659039.0   \n",
       "33183019   https://doi.org/10.21037/apm-20-1485; https://...  226855189.0   \n",
       "32549917   https://doi.org/10.1007/s13167-020-00205-2; ht...  218624800.0   \n",
       "29938891   https://doi.org/10.1111/obr.12692; https://www...   49418389.0   \n",
       "33406963   https://doi.org/10.1177/0733464820983976; http...  230817520.0   \n",
       "33236629   https://doi.org/10.1021/acs.analchem.0c03223; ...  227167582.0   \n",
       "27490600   https://doi.org/10.1097/pcc.0000000000000780; ...   40483525.0   \n",
       "31991422   https://doi.org/10.1159/000504805; https://www...  210945568.0   \n",
       "32409432   https://doi.org/10.3949/ccjm.87a.ccc028; https...  218649040.0   \n",
       "19477503   https://doi.org/10.1016/s0140-6736(09)60941-7;...   11718220.0   \n",
       "32816516   https://doi.org/10.1037/tra0000903; https://ww...  221222419.0   \n",
       "32802446   https://doi.org/10.21037/jtd-20-955; https://w...  225756814.0   \n",
       "33351483   https://doi.org/10.1097/sla.0000000000004678; ...  229696117.0   \n",
       "33044493   https://doi.org/10.1001/jamapediatrics.2020.42...  222302311.0   \n",
       "32443114   https://doi.org/10.1097/01.naj.0000668700.2300...  218858413.0   \n",
       "33757196   https://doi.org/10.3934/mbe.2021078; https://w...  232340398.0   \n",
       "\n",
       "                       label  \\\n",
       "pubmed_id                      \n",
       "32572175     primary-not-rct   \n",
       "33079612            excluded   \n",
       "33565705     primary-not-rct   \n",
       "33479162     primary-not-rct   \n",
       "33705595            excluded   \n",
       "33771818     primary-not-rct   \n",
       "32890322     broad-synthesis   \n",
       "33381888     primary-not-rct   \n",
       "32628408            excluded   \n",
       "33179469     primary-not-rct   \n",
       "33305712            excluded   \n",
       "32920815     primary-not-rct   \n",
       "32657602     primary-not-rct   \n",
       "32601125            excluded   \n",
       "33620014            excluded   \n",
       "33533477            excluded   \n",
       "33290467            excluded   \n",
       "33645461     primary-not-rct   \n",
       "33004394   systematic-review   \n",
       "33739446     primary-not-rct   \n",
       "33557718     primary-not-rct   \n",
       "32459567            excluded   \n",
       "33615885            excluded   \n",
       "33315291            excluded   \n",
       "32866992            excluded   \n",
       "28958212   systematic-review   \n",
       "33647224     primary-not-rct   \n",
       "33770334     primary-not-rct   \n",
       "33569794     primary-not-rct   \n",
       "33368849     primary-not-rct   \n",
       "...                      ...   \n",
       "33283686     primary-not-rct   \n",
       "33730407     primary-not-rct   \n",
       "28840782     primary-not-rct   \n",
       "32419599     primary-not-rct   \n",
       "33526471            excluded   \n",
       "31920368   systematic-review   \n",
       "33141520            excluded   \n",
       "33085086     primary-not-rct   \n",
       "33744954     primary-not-rct   \n",
       "32571147            excluded   \n",
       "33123349     primary-not-rct   \n",
       "32350226     primary-not-rct   \n",
       "33530819            excluded   \n",
       "33646036            excluded   \n",
       "32413164            excluded   \n",
       "33183019     primary-not-rct   \n",
       "32549917   systematic-review   \n",
       "29938891   systematic-review   \n",
       "33406963     primary-not-rct   \n",
       "33236629            excluded   \n",
       "27490600   systematic-review   \n",
       "31991422   systematic-review   \n",
       "32409432     primary-not-rct   \n",
       "19477503         primary-rct   \n",
       "32816516     primary-not-rct   \n",
       "32802446     primary-not-rct   \n",
       "33351483     primary-not-rct   \n",
       "33044493     primary-not-rct   \n",
       "32443114     primary-not-rct   \n",
       "33757196     primary-not-rct   \n",
       "\n",
       "                                                    document  \\\n",
       "pubmed_id                                                      \n",
       "32572175   The end of social confinement and COVID-19 re-...   \n",
       "33079612   The Missing Link in the Covid-19 Vaccine Race....   \n",
       "33565705   Donor To Recipient Transmission Of SARS-CoV-2 ...   \n",
       "33479162   Neonates Born to Mothers With COVID-19: Data F...   \n",
       "33705595   New targets for drug design: Importance of nsp...   \n",
       "33771818   Establishing a COVID-19 treatment centre in Is...   \n",
       "32890322   Guidelines for Ultrasound in the Radiology Dep...   \n",
       "33381888   Antiphospholipid antibodies and thrombosis in ...   \n",
       "32628408   Nonoperating room anesthesia for patients with...   \n",
       "33179469   Surgical management protocol during the COVID-...   \n",
       "33305712   COVID-19 Severity: Lung-Heart Interplay. In De...   \n",
       "32920815   Novel coronavirus(Covid19), Its renal manifest...   \n",
       "32657602   The COVID-19 Outbreak in Iran; The First Patie...   \n",
       "32601125   Potential role of endothelial cell surface ect...   \n",
       "33620014   Convalescent Plasma and Hyperimmune Globulin T...   \n",
       "33533477   Nitric oxide and Covid-19: dose, timing and ho...   \n",
       "33290467   Introductory lecture: air quality in megacitie...   \n",
       "33645461   Pneumocystis jirovecii pneumonia in an immunoc...   \n",
       "33004394   Nutrition screening and assessment tools for p...   \n",
       "33739446   Left-behind children's social adjustment and r...   \n",
       "33557718   Social work leadership in a medical school: A ...   \n",
       "32459567   Use of systemic therapies for psoriasis in the...   \n",
       "33615885   Do recipients of the first dose of the Pfizer/...   \n",
       "33315291   Does inflammation link stress to poor COVID-19...   \n",
       "32866992   Paucity of COVID-19 Dermatology Literature fro...   \n",
       "28958212   Effectiveness of remote feedback on physical a...   \n",
       "33647224   Results of preoperative screening for COVID-19...   \n",
       "33770334   The Impact of COVID-19 on Access to HIV Preven...   \n",
       "33569794   New thing, The Impact of the Emergence of COVI...   \n",
       "33368849   Perceived stress and social support in a large...   \n",
       "...                                                      ...   \n",
       "33283686   Effect of coronaphobia on smoking habits. INTR...   \n",
       "33730407   A simple, safe and sensitive method for SARS-C...   \n",
       "28840782   Comparison of three blood transfusion guidelin...   \n",
       "32419599   Are there positive lessons for Italy's NHS res...   \n",
       "33526471   A Novel Soluble ACE2 Variant with Prolonged Du...   \n",
       "31920368   Nonpharmacological Interventions Addressing Pa...   \n",
       "33141520   Proteomic Biomarkers to Guide Stratification f...   \n",
       "33085086   Increased occurrence of hand eczema in young c...   \n",
       "33744954   Anti-SARS-CoV2 antibody responses in serum and...   \n",
       "32571147   The Difficult Airway and Aerosol-Generating Pr...   \n",
       "33123349   In vitro efficacy of a copper iodine complex P...   \n",
       "32350226   An assessment of real-time RT-PCR kits for SAR...   \n",
       "33530819   Peripheral blood microRNAs and the COVID-19 pa...   \n",
       "33646036   COVID-19 vaccines: what happened to evidence-b...   \n",
       "32413164   Delivering psychotherapy by video conference i...   \n",
       "33183019   Complete and enduring response in an elderly p...   \n",
       "32549917   Sleep disorders in migrants and refugees: a sy...   \n",
       "29938891   Measuring beverage consumption in US children ...   \n",
       "33406963   Brazilian Older People Hospitalized by COVID-1...   \n",
       "33236629   Specificity Enhancement of Deoxyribonucleic Ac...   \n",
       "27490600   Near Infrared Spectroscopy as a Hemodynamic Mo...   \n",
       "31991422   Epidural-Related Fever and Maternal and Neonat...   \n",
       "32409432   Home monitoring for COVID-19. Cleveland Clinic...   \n",
       "19477503   Effectiveness of thigh-length graduated compre...   \n",
       "32816516   What the world could learn from the Haitian re...   \n",
       "32802446   Investigation of a family cluster outbreak of ...   \n",
       "33351483   Yield of Adding chest CT to Abdominal CT to De...   \n",
       "33044493   Outcomes of Neonates Born to Mothers With Seve...   \n",
       "32443114   Where Are the Masks? Shortages of vital PPE du...   \n",
       "33757196   A deep learning-based medication behavior moni...   \n",
       "\n",
       "                                                    features  \\\n",
       "pubmed_id                                                      \n",
       "32572175   [101, 1103, 1322, 1104, 1934, 24478, 1105, 188...   \n",
       "33079612   [101, 1103, 3764, 5088, 1107, 1103, 1884, 1831...   \n",
       "33565705   [101, 16667, 1106, 7668, 6580, 1104, 21718, 17...   \n",
       "33479162   [101, 24762, 5430, 1255, 1106, 12694, 1114, 18...   \n",
       "33705595   [101, 1207, 7539, 1111, 3850, 1902, 131, 4495,...   \n",
       "33771818   [101, 7046, 170, 1884, 18312, 118, 1627, 3252,...   \n",
       "32890322   [101, 13112, 1111, 18737, 22909, 1107, 1103, 2...   \n",
       "33381888   [101, 2848, 7880, 2155, 7880, 11014, 25786, 26...   \n",
       "32628408   [101, 1664, 19807, 3798, 1395, 1126, 2556, 273...   \n",
       "33179469   [101, 13467, 2635, 11309, 1219, 1103, 1884, 18...   \n",
       "33305712   [101, 1884, 18312, 118, 1627, 20997, 131, 1309...   \n",
       "32920815   [101, 2281, 1884, 15789, 27608, 113, 1884, 183...   \n",
       "32657602   [101, 1103, 1884, 18312, 118, 1627, 8010, 1107...   \n",
       "32601125   [101, 3209, 1648, 1104, 1322, 12858, 21091, 13...   \n",
       "33620014   [101, 14255, 18236, 23193, 13441, 1105, 177, 2...   \n",
       "33533477   [101, 11437, 11048, 17151, 1105, 1884, 18312, ...   \n",
       "33290467   [101, 27553, 26739, 10309, 131, 1586, 3068, 11...   \n",
       "33645461   [101, 185, 1673, 1818, 13335, 6834, 6620, 2322...   \n",
       "33004394   [101, 20121, 11954, 1105, 8670, 5537, 1111, 44...   \n",
       "33739446   [101, 1286, 118, 1481, 1482, 112, 188, 1934, 2...   \n",
       "33557718   [101, 1934, 1250, 3645, 1107, 170, 2657, 1278,...   \n",
       "32459567   [101, 1329, 1104, 27410, 1103, 14543, 1905, 11...   \n",
       "33615885   [101, 1202, 18509, 1104, 1103, 1148, 13753, 11...   \n",
       "33315291   [101, 1674, 24970, 5088, 6600, 1106, 2869, 188...   \n",
       "32866992   [101, 185, 3984, 9041, 1104, 1884, 18312, 118,...   \n",
       "28958212   [101, 12949, 1104, 6456, 13032, 1113, 2952, 32...   \n",
       "33647224   [101, 2686, 1104, 3073, 19807, 5838, 11954, 11...   \n",
       "33770334   [101, 1103, 3772, 1104, 1884, 18312, 118, 1627...   \n",
       "33569794   [101, 1207, 1645, 117, 1103, 3772, 1104, 1103,...   \n",
       "33368849   [101, 8367, 6600, 1105, 1934, 1619, 1107, 170,...   \n",
       "...                                                      ...   \n",
       "33283686   [101, 2629, 1104, 1884, 15789, 7880, 23145, 11...   \n",
       "33730407   [101, 170, 3014, 117, 2914, 1105, 7246, 3442, ...   \n",
       "28840782   [101, 7577, 1104, 1210, 1892, 14715, 17149, 13...   \n",
       "32419599   [101, 1132, 1175, 3112, 8497, 1111, 1122, 1744...   \n",
       "33526471   [101, 170, 2281, 28043, 20839, 1477, 8120, 111...   \n",
       "31920368   [101, 1664, 20695, 10841, 23388, 22496, 13735,...   \n",
       "33141520   [101, 5250, 1566, 18882, 1665, 25128, 8519, 14...   \n",
       "33085086   [101, 2569, 15299, 1104, 1289, 174, 18587, 144...   \n",
       "33744954   [101, 2848, 118, 21718, 1733, 118, 1884, 1964,...   \n",
       "32571147   [101, 1103, 2846, 1586, 2787, 1105, 170, 10771...   \n",
       "33123349   [101, 1107, 191, 2875, 2180, 23891, 1104, 170,...   \n",
       "32350226   [101, 1126, 8670, 1104, 1842, 118, 1159, 187, ...   \n",
       "33530819   [101, 17963, 1892, 17599, 11782, 1116, 1105, 1...   \n",
       "33646036   [101, 1884, 18312, 118, 1627, 20034, 1116, 131...   \n",
       "32413164   [101, 13232, 15604, 21155, 20939, 1118, 1888, ...   \n",
       "33183019   [101, 2335, 1105, 21585, 2593, 1107, 1126, 980...   \n",
       "32549917   [101, 2946, 11759, 1107, 18677, 1105, 8940, 13...   \n",
       "29938891   [101, 10099, 24802, 8160, 1107, 1366, 1482, 11...   \n",
       "33406963   [101, 12418, 5303, 15647, 2214, 1234, 2704, 22...   \n",
       "33236629   [101, 2747, 1785, 11778, 1880, 1104, 1260, 106...   \n",
       "27490600   [101, 1485, 18688, 188, 26426, 5864, 20739, 11...   \n",
       "31991422   [101, 174, 25786, 12602, 118, 2272, 10880, 110...   \n",
       "32409432   [101, 1313, 9437, 1111, 1884, 18312, 118, 1627...   \n",
       "19477503   [101, 12949, 1104, 9973, 118, 2251, 3024, 1492...   \n",
       "32816516   [101, 1184, 1103, 1362, 1180, 3858, 1121, 1103...   \n",
       "32802446   [101, 4449, 1104, 170, 1266, 10005, 8010, 1104...   \n",
       "33351483   [101, 10972, 1104, 5321, 2229, 172, 1204, 1106...   \n",
       "33044493   [101, 13950, 1104, 24762, 5430, 1255, 1106, 12...   \n",
       "32443114   [101, 1187, 1132, 1103, 17944, 136, 25630, 110...   \n",
       "33757196   [101, 170, 1996, 3776, 118, 1359, 15683, 4658,...   \n",
       "\n",
       "                                                       masks  \n",
       "pubmed_id                                                     \n",
       "32572175   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33079612   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33565705   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33479162   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33705595   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33771818   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32890322   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33381888   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32628408   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33179469   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33305712   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32920815   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32657602   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32601125   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33620014   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33533477   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33290467   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33645461   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33004394   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33739446   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33557718   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32459567   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33615885   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33315291   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32866992   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "28958212   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33647224   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33770334   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33569794   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33368849   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "...                                                      ...  \n",
       "33283686   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33730407   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "28840782   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32419599   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33526471   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "31920368   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33141520   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33085086   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33744954   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32571147   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33123349   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32350226   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33530819   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33646036   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32413164   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33183019   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32549917   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "29938891   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33406963   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33236629   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "27490600   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "31991422   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32409432   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "19477503   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32816516   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32802446   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33351483   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33044493   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "32443114   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "33757196   [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...  \n",
       "\n",
       "[18854 rows x 23 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load df CORD19 document types  \n",
    "df_test = pd.read_csv('path/to/test/CORD19_full_labels.csv', sep='\\t')\n",
    "\n",
    "# drop document with no abstract\n",
    "df_test.dropna(subset = ['title', 'abstract'], inplace=True)\n",
    "\n",
    "df_test['document'] = [x + ' ' + y for x,y in zip(df_test.title, df_test.abstract)]\n",
    "\n",
    "df_test.index = df_test['pubmed_id']\n",
    "\n",
    "# create features and mask columns \n",
    "text_list = df_test[\"document\"].values\n",
    "input_ids = tokenize_inputs(text_list, tokenizer, num_embeddings = 500)\n",
    "attention_masks = create_attn_masks(input_ids)\n",
    "\n",
    "# add input ids and attention masks to the dataframe\n",
    "df_test[\"features\"] = input_ids.tolist()\n",
    "df_test[\"masks\"] = attention_masks\n",
    "\n",
    "\n",
    "df_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:01:09.944591Z",
     "start_time": "2021-05-25T20:01:09.940152Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "primary-not-rct      9110\n",
       "excluded             5634\n",
       "systematic-review    3380\n",
       "broad-synthesis       492\n",
       "primary-rct           238\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:08:49.580472Z",
     "start_time": "2021-05-25T20:01:09.945602Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18853/18854\r"
     ]
    }
   ],
   "source": [
    "def generate_predictions(model, df, num_labels, device=\"cpu\", batch_size=32):\n",
    "    \n",
    "    num_iter = math.ceil(df.shape[0]/batch_size)\n",
    "    pred_probs = np.array([]).reshape(0, num_labels)\n",
    "    \n",
    "    model.to(device)\n",
    "    model.eval()\n",
    "    \n",
    "    for i in range(num_iter):\n",
    "        \n",
    "        print('{}/{}'.format(i, num_iter), end='\\r')\n",
    "        \n",
    "        df_subset = df.iloc[i*batch_size:(i+1)*batch_size,:]\n",
    "        X = df_subset[\"features\"].values.tolist()\n",
    "        \n",
    "        masks = df_subset[\"masks\"].values.tolist()\n",
    "        X = torch.tensor(X)\n",
    "        \n",
    "        masks = torch.tensor(masks, dtype=torch.long)\n",
    "        \n",
    "        X = X.to(device)\n",
    "        masks = masks.to(device)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            \n",
    "            logits = model(input_ids=X, attention_mask=masks)\n",
    "            logits = logits.sigmoid().detach().cpu().numpy()\n",
    "            pred_probs = np.vstack([pred_probs, logits])\n",
    " \n",
    "    return pred_probs \n",
    "\n",
    "num_labels = 5\n",
    "\n",
    "pred_probs = generate_predictions(model, df_test, num_labels, device=\"cuda\", batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:09:08.106823Z",
     "start_time": "2021-05-25T20:09:08.103343Z"
    }
   },
   "outputs": [],
   "source": [
    "predictions = np.argmax(pred_probs, axis=1)\n",
    "\n",
    "df_test['pred'] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:09:12.141418Z",
     "start_time": "2021-05-25T20:09:12.132595Z"
    }
   },
   "outputs": [],
   "source": [
    "gt = []\n",
    "\n",
    "for x in df_test.label:\n",
    "    \n",
    "    if x == 'systematic-review':\n",
    "        gt.append(4)\n",
    "    \n",
    "    elif x == 'primary-not-rct':\n",
    "        gt.append(3)\n",
    "    \n",
    "    elif x == 'primary-rct':\n",
    "        gt.append(2)\n",
    "    \n",
    "    elif x == 'excluded':\n",
    "        gt.append(1)\n",
    "    \n",
    "    elif x == 'broad-synthesis':\n",
    "        gt.append(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:09:13.397587Z",
     "start_time": "2021-05-25T20:09:13.389986Z"
    }
   },
   "outputs": [],
   "source": [
    "df_test['ground_truth'] = gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:09:14.022713Z",
     "start_time": "2021-05-25T20:09:14.017993Z"
    }
   },
   "outputs": [],
   "source": [
    "gt = np.array(df_test.ground_truth)\n",
    "preds = np.array(df_test.pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-25T20:09:21.091986Z",
     "start_time": "2021-05-25T20:09:21.082885Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.56      0.69      0.62       492\n",
      "          1       0.90      0.62      0.73      5634\n",
      "          2       0.64      0.80      0.71       238\n",
      "          3       0.82      0.96      0.88      9110\n",
      "          4       0.94      0.92      0.93      3380\n",
      "\n",
      "avg / total       0.85      0.84      0.84     18854\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(gt, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}