extract-delays.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract delays from PubMed history dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "library(dplyr, warn=F)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>pubmed_id</th><th scope=col>journal_nlm_id</th><th scope=col>medline</th><th scope=col>pubmed</th><th scope=col>entrez_0</th><th scope=col>date_online</th><th scope=col>aheadofprint_0</th><th scope=col>received</th><th scope=col>accepted</th><th scope=col>revised_0</th><th scope=col>epublish_0</th><th scope=col>pmc-release_0</th><th scope=col>ecollection_0</th><th scope=col>ppublish_0</th><th scope=col>version_0</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>1</td><td>0151424</td><td>1975-06-01</td><td>1975-06-01</td><td>1975-06-01</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>2</td><td>0372516</td><td>1975-10-27</td><td>1975-10-27</td><td>1975-10-27</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllllllllllll}\n",
       "  & pubmed_id & journal_nlm_id & medline & pubmed & entrez_0 & date_online & aheadofprint_0 & received & accepted & revised_0 & epublish_0 & pmc-release_0 & ecollection_0 & ppublish_0 & version_0\\\\\n",
       "\\hline\n",
       "\t1 & 1 & 0151424 & 1975-06-01 & 1975-06-01 & 1975-06-01 & NA & NA & NA & NA & NA & NA & NA & NA & NA & NA\\\\\n",
       "\t2 & 2 & 0372516 & 1975-10-27 & 1975-10-27 & 1975-10-27 & NA & NA & NA & NA & NA & NA & NA & NA & NA & NA\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [2 x 15]\n",
       "\n",
       "  pubmed_id journal_nlm_id    medline     pubmed   entrez_0 date_online\n",
       "      (int)          (chr)     (date)     (date)     (date)      (date)\n",
       "1         1        0151424 1975-06-01 1975-06-01 1975-06-01        <NA>\n",
       "2         2        0372516 1975-10-27 1975-10-27 1975-10-27        <NA>\n",
       "Variables not shown: aheadofprint_0 (chr), received (date), accepted (date),\n",
       "  revised_0 (chr), epublish_0 (chr), pmc-release_0 (chr), ecollection_0 (chr),\n",
       "  ppublish_0 (chr), version_0 (chr)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read history dates for all articles\n",
    "path = file.path('data', 'history-dates.tsv.bz2')\n",
    "col_types = list(\n",
    "  accepted_0 = readr::col_date(),\n",
    "  received_0 = readr::col_date(),\n",
    "  pubmed_0 = readr::col_date(),\n",
    "  medline_0 = readr::col_date(),\n",
    "  date_online = readr::col_date()\n",
    ")\n",
    "all_df = readr::read_tsv(path, col_types = col_types) %>%\n",
    "  dplyr::rename(received = received_0, accepted = accepted_0, pubmed = pubmed_0, medline = medline_0)\n",
    "head(all_df, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set time constraint to elimate erroneous records\n",
    "earliest = readr::parse_date('1960-01-01')\n",
    "latest = readr::parse_date('2015-12-31')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>year</th><th scope=col>n_journals</th><th scope=col>n_articles</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>1960</td><td>1925</td><td>111955</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>1961</td><td>2440</td><td>119933</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lll}\n",
       "  & year & n_journals & n_articles\\\\\n",
       "\\hline\n",
       "\t1 & 1960 & 1925 & 111955\\\\\n",
       "\t2 & 1961 & 2440 & 119933\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [2 x 3]\n",
       "\n",
       "   year n_journals n_articles\n",
       "  (dbl)      (int)      (int)\n",
       "1  1960       1925     111955\n",
       "2  1961       2440     119933"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count journals and articles by PubMed year\n",
    "year_df = all_df %>%\n",
    "  dplyr::filter(pubmed >= earliest) %>%\n",
    "  dplyr::filter(pubmed <= latest) %>%\n",
    "  dplyr::mutate(year = lubridate::year(pubmed)) %>%\n",
    "  dplyr::group_by(year) %>%\n",
    "  dplyr::summarize(\n",
    "    n_journals = n_distinct(journal_nlm_id),\n",
    "    n_articles = n()\n",
    "  )\n",
    "\n",
    "path = file.path('data', 'yearly-pubmed-totals.tsv')\n",
    "year_df %>%\n",
    "  readr::write_tsv(path)\n",
    "\n",
    "head(year_df, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "3330333"
      ],
      "text/latex": [
       "3330333"
      ],
      "text/markdown": [
       "3330333"
      ],
      "text/plain": [
       "[1] 3330333"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create an acceptance  delay dataset\n",
    "accept_df = all_df %>%\n",
    "  dplyr::mutate(delay_type = 'Acceptance') %>%\n",
    "  dplyr::mutate(delay = as.numeric(accepted - received, units='days')) %>%\n",
    "  dplyr::rename(date = accepted) %>%\n",
    "  dplyr::select(journal_nlm_id, pubmed_id, delay_type, date, delay) %>%\n",
    "  dplyr::filter(! is.na(delay)) %>%\n",
    "  dplyr::filter(delay > 0) %>%\n",
    "  dplyr::filter(delay <= 365 * 5) %>%\n",
    "  dplyr::filter(date >= earliest) %>%\n",
    "  dplyr::filter(date <= latest)\n",
    "\n",
    "nrow(accept_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal_nlm_id</th><th scope=col>pubmed_id</th><th scope=col>delay_type</th><th scope=col>date</th><th scope=col>delay</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>8214379</td><td>2408592</td><td>Acceptance</td><td>1984-12-07</td><td>2</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>8214379</td><td>2412502</td><td>Acceptance</td><td>1985-03-21</td><td>55</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       "  & journal_nlm_id & pubmed_id & delay_type & date & delay\\\\\n",
       "\\hline\n",
       "\t1 & 8214379 & 2408592 & Acceptance & 1984-12-07 & 2\\\\\n",
       "\t2 & 8214379 & 2412502 & Acceptance & 1985-03-21 & 55\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [2 x 5]\n",
       "\n",
       "  journal_nlm_id pubmed_id delay_type       date delay\n",
       "           (chr)     (int)      (chr)     (date) (dbl)\n",
       "1        8214379   2408592 Acceptance 1984-12-07     2\n",
       "2        8214379   2412502 Acceptance 1985-03-21    55"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head(accept_df, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "2765750"
      ],
      "text/latex": [
       "2765750"
      ],
      "text/markdown": [
       "2765750"
      ],
      "text/plain": [
       "[1] 2765750"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a publication delay dataset\n",
    "publish_df = all_df %>%\n",
    "  dplyr::mutate(delay_type = 'Publication') %>%\n",
    "  dplyr::mutate(delay = as.numeric(date_online - accepted, units='days')) %>%\n",
    "  dplyr::rename(date = date_online) %>%\n",
    "  dplyr::select(journal_nlm_id, pubmed_id, delay_type, date, delay) %>%\n",
    "  dplyr::filter(! is.na(delay)) %>%\n",
    "  dplyr::filter(delay >= 0) %>%\n",
    "  dplyr::filter(delay <= 365 * 3) %>%\n",
    "  dplyr::filter(date >= earliest) %>%\n",
    "  dplyr::filter(date <= latest)\n",
    "\n",
    "nrow(publish_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal_nlm_id</th><th scope=col>pubmed_id</th><th scope=col>delay_type</th><th scope=col>date</th><th scope=col>delay</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>9305878</td><td>10089389</td><td>Publication</td><td>1999-01-01</td><td>74</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>9305878</td><td>10089390</td><td>Publication</td><td>1999-01-01</td><td>247</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       "  & journal_nlm_id & pubmed_id & delay_type & date & delay\\\\\n",
       "\\hline\n",
       "\t1 & 9305878 & 10089389 & Publication & 1999-01-01 & 74\\\\\n",
       "\t2 & 9305878 & 10089390 & Publication & 1999-01-01 & 247\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [2 x 5]\n",
       "\n",
       "  journal_nlm_id pubmed_id  delay_type       date delay\n",
       "           (chr)     (int)       (chr)     (date) (dbl)\n",
       "1        9305878  10089389 Publication 1999-01-01    74\n",
       "2        9305878  10089390 Publication 1999-01-01   247"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head(publish_df, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Bind acceptance and publication dataframes\n",
    "delay_df = dplyr::bind_rows(accept_df, publish_df) %>%\n",
    "  dplyr::arrange(journal_nlm_id, pubmed_id, delay_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal_nlm_id</th><th scope=col>pubmed_id</th><th scope=col>delay_type</th><th scope=col>date</th><th scope=col>delay</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>0001027</td><td>22221113</td><td>Acceptance</td><td>2011-11-15</td><td>111</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>0001027</td><td>22221113</td><td>Publication</td><td>2012-01-05</td><td>51</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>0001027</td><td>22221154</td><td>Acceptance</td><td>2011-11-15</td><td>227</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>0001027</td><td>22221154</td><td>Publication</td><td>2012-01-05</td><td>51</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>0001027</td><td>22224504</td><td>Acceptance</td><td>2011-11-15</td><td>88</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>0001027</td><td>22224504</td><td>Publication</td><td>2012-01-08</td><td>54</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       "  & journal_nlm_id & pubmed_id & delay_type & date & delay\\\\\n",
       "\\hline\n",
       "\t1 & 0001027 & 22221113 & Acceptance & 2011-11-15 & 111\\\\\n",
       "\t2 & 0001027 & 22221113 & Publication & 2012-01-05 & 51\\\\\n",
       "\t3 & 0001027 & 22221154 & Acceptance & 2011-11-15 & 227\\\\\n",
       "\t4 & 0001027 & 22221154 & Publication & 2012-01-05 & 51\\\\\n",
       "\t5 & 0001027 & 22224504 & Acceptance & 2011-11-15 & 88\\\\\n",
       "\t6 & 0001027 & 22224504 & Publication & 2012-01-08 & 54\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [6 x 5]\n",
       "\n",
       "  journal_nlm_id pubmed_id  delay_type       date delay\n",
       "           (chr)     (int)       (chr)     (date) (dbl)\n",
       "1        0001027  22221113  Acceptance 2011-11-15   111\n",
       "2        0001027  22221113 Publication 2012-01-05    51\n",
       "3        0001027  22221154  Acceptance 2011-11-15   227\n",
       "4        0001027  22221154 Publication 2012-01-05    51\n",
       "5        0001027  22224504  Acceptance 2011-11-15    88\n",
       "6        0001027  22224504 Publication 2012-01-08    54"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "head(delay_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>journal_nlm_id</th><th scope=col>pubmed_id</th><th scope=col>delay_type</th><th scope=col>date</th><th scope=col>delay</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>1</th><td>9892366</td><td>21423322</td><td>Acceptance</td><td>2010-08-15</td><td>90</td></tr>\n",
       "\t<tr><th scope=row>2</th><td>9892366</td><td>21423322</td><td>Publication</td><td>2010-09-01</td><td>17</td></tr>\n",
       "\t<tr><th scope=row>3</th><td>9892366</td><td>26097404</td><td>Acceptance</td><td>2014-11-19</td><td>168</td></tr>\n",
       "\t<tr><th scope=row>4</th><td>9892366</td><td>26097404</td><td>Publication</td><td>2015-01-10</td><td>52</td></tr>\n",
       "\t<tr><th scope=row>5</th><td>9892366</td><td>26321875</td><td>Acceptance</td><td>2013-01-08</td><td>100</td></tr>\n",
       "\t<tr><th scope=row>6</th><td>9892366</td><td>26321875</td><td>Publication</td><td>2013-02-07</td><td>30</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       "  & journal_nlm_id & pubmed_id & delay_type & date & delay\\\\\n",
       "\\hline\n",
       "\t1 & 9892366 & 21423322 & Acceptance & 2010-08-15 & 90\\\\\n",
       "\t2 & 9892366 & 21423322 & Publication & 2010-09-01 & 17\\\\\n",
       "\t3 & 9892366 & 26097404 & Acceptance & 2014-11-19 & 168\\\\\n",
       "\t4 & 9892366 & 26097404 & Publication & 2015-01-10 & 52\\\\\n",
       "\t5 & 9892366 & 26321875 & Acceptance & 2013-01-08 & 100\\\\\n",
       "\t6 & 9892366 & 26321875 & Publication & 2013-02-07 & 30\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "Source: local data frame [6 x 5]\n",
       "\n",
       "  journal_nlm_id pubmed_id  delay_type       date delay\n",
       "           (chr)     (int)       (chr)     (date) (dbl)\n",
       "1        9892366  21423322  Acceptance 2010-08-15    90\n",
       "2        9892366  21423322 Publication 2010-09-01    17\n",
       "3        9892366  26097404  Acceptance 2014-11-19   168\n",
       "4        9892366  26097404 Publication 2015-01-10    52\n",
       "5        9892366  26321875  Acceptance 2013-01-08   100\n",
       "6        9892366  26321875 Publication 2013-02-07    30"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tail(delay_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Save as a gzipped TSV\n",
    "path = file.path('data', 'delays.tsv')\n",
    "delay_df %>%\n",
    "  readr::write_tsv(path)\n",
    "system2('gzip', c('--force', path))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}