Figure_1B.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "#Due to leftoevererrors in Nathan's python installation, some cleaning up occurs here\n",
    "#sys.path.append(\"./code/\")\n",
    "#sys.path.remove('/usr/local/lib/python2.7/site-packages') \n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import copy\n",
    "import datetime\n",
    "import pickle\n",
    "import gzip\n",
    "import os,glob\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import random\n",
    "from  scipy import stats\n",
    "\n",
    "\n",
    "\n",
    "from scipy.stats import norm\n",
    "from sklearn.neighbors import KernelDensity\n",
    "\n",
    "\n",
    "import itertools\n",
    "#sys.path\n",
    "\n",
    "\n",
    "import regex as re\n",
    "import datetime\n",
    "import math\n",
    "import time\n",
    "import itertools\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "from plotly import tools\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "import plotly\n",
    "plotly.tools.set_credentials_file(username='juliettapc', api_key='nM6iUdx6dGaOiPXQTwpP')   # go to: https://plot.ly/settings/api#/   for a new key if needed\n",
    "\n",
    "########## to be able to plot offline (without sending the plots to the plotly server every time)\n",
    "import plotly.offline as offline\n",
    "from plotly.graph_objs import *\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "\n",
    "init_notebook_mode(connected=True)\n",
    "################\n",
    "\n",
    "\n",
    "\n",
    "from IPython.core.display import display,HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))  # to make the notebook use the entire width of the browser\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "                                  \n",
    "\n",
    "####  NEW FILE INCLUDING EARLY CITATIONS OF YOUNG REFERENCES:   ../data/df_reference_cite_plos_no_self-cit_one_ref_per_sect_ONLY_ARTICLES_early_cit.pkl\n",
    "                                            \n",
    "#%time df_merged = pickle.load(open('../data/df_reference_cite_plos_merged_simplified_added_more_columns_no_self-cit_one_ref_per_sect_ONLY_ARTICLES.pkl', 'rb'))\n",
    "\n",
    "\n",
    "\n",
    "%time df_merged = pickle.load(open('../data/df_reference_cite_plos_no_self-cit_one_ref_per_sect_ONLY_ARTICLES_early_cit_and_after_accretion_time.pkl', 'rb'))\n",
    "print (\"done loading pickles\", df_merged.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "df_merged = df_merged[df_merged['cite_count'] != -1]   # i dont know why, but there are 7 occurrences with value -1\n",
    "\n",
    "\n",
    "\n",
    "plos_df = df_merged.drop_duplicates(subset=['paper_UT'])\n",
    "print (plos_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "df_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " ##### Histogram number of occurrences in each section (all papers)\n",
    "#### FIGURE 1B\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "v1_string = 'regex_sect_index'\n",
    "       \n",
    "\n",
    "years=[2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017] \n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "string_filtering_x = 'paper_cite_count'   # bins by plos' citations on the x-axis ###      \n",
    "  \n",
    "string_references_age = \"\"   #young\"#old\"  # young # all   for the selection of what references i include\n",
    "    \n",
    "string_isolated_ref = \"\"   #\"\"   #\"  #\"\"   # 0  or 1 (or empty string, to include all ref)\n",
    "  \n",
    "string_self_ref =0    #\"\"      # \"\"   #1   # 0  or 1 (or empty string, to include all ref)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### plos ONE categories. \n",
    "string_code_categ=\"\" #  ojo!!! the codes are strings, not integers. if i want to include multiple subjects:  \"1 2 8\"\n",
    " \n",
    "#  '0': 'Biology and life sciences'             6,032,537\n",
    "#  '1': 'Computer and information sciences'     1,207,799\n",
    "#  '10': 'Social sciences'                      755,899\n",
    "#  '2': 'Earth sciences'                        533,155\n",
    "#  '3': 'Ecology and environmental sciences'    624,142\n",
    "#  '4': 'Engineering and technology'            382,247 \n",
    "#  '5': 'Medicine and health sciences'          4,535,926   \n",
    "#  '6': 'People and places'                     691,523\n",
    "#  '7': 'Physical sciences'                     2,100,827\n",
    "#  '8': 'Research and analysis methods'         3,871,470\n",
    "#  '9': 'Science policy'                        43,360 \n",
    "\n",
    "\n",
    "######### plos journals \n",
    "string_journal=\"\"\n",
    "\n",
    "    # PLOS ONE       6,367,070\n",
    "    # PLOS GENET      149,923\n",
    "    # PLO NE TR D     138,289   # (neglected tropical diseases)\n",
    "    # PLOS PATHOG     109,803\n",
    "    # PLOS COMPUT      77,924\n",
    "    # PLOS BIOL        56,754\n",
    "    # PLOS MED         24,506\n",
    "\n",
    " \n",
    "\n",
    "######### WoS subject categories. \n",
    "string_plos_field=\"\"#['D CU BIOLOGY']\"\n",
    "\n",
    "# ['D RO MULTIDISCIPLINARY SCIENCES']                                                                                                       4464540\n",
    "# ['D CU BIOLOGY']                                                                                                                          1055045\n",
    "# ['D RO MULTIDISCIPLINARY SCIENCES', 'D CU BIOLOGY']                                                                                        847485\n",
    "# ['D KM GENETICS & HEREDITY']                                                                                                               149923\n",
    "# ['D YU TROPICAL MEDICINE', 'D TI PARASITOLOGY']                                                                                            138289\n",
    "# ['D ZE VIROLOGY', 'D QU MICROBIOLOGY', 'D TI PARASITOLOGY']                                                                                109803\n",
    "# ['D CO BIOCHEMICAL RESEARCH METHODS', 'D MC MATHEMATICAL & COMPUTATIONAL BIOLOGY']                                                          77687\n",
    "# ['D CQ BIOCHEMISTRY & MOLECULAR BIOLOGY', 'D CU BIOLOGY']                                                                                   56754\n",
    "# ['D PY MEDICINE, GENERAL & INTERNAL']                                                                                                       24506\n",
    "# ['D CO BIOCHEMICAL RESEARCH METHODS', 'D MC MATHEMATICAL & COMPUTATIONAL BIOLOGY', 'D PO MATHEMATICS, INTERDISCIPLINARY APPLICATIONS']        237\n",
    "\n",
    "\n",
    "    \n",
    "print (\"original size:\",df_merged.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "##### preselection by plos year\n",
    "print (years)\n",
    "preselection_df = df_merged[df_merged['plos_pub_year'].isin(years)]  \n",
    "print (\"size of preselection1 (by plos years):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#### i remove self-citations\n",
    "if (string_self_ref==0) or  ( string_self_ref == 1 ): \n",
    "    preselection_df = preselection_df[preselection_df['self_citation']== string_self_ref ]  \n",
    "    if string_self_ref ==0:\n",
    "        string_self_ref = \", no self-cit\"\n",
    "    elif string_self_ref ==1:\n",
    "        string_self_ref = \", only self-cit\"\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "        \n",
    "######### preselection by isolated or group references:\n",
    "if (string_isolated_ref==0) or  ( string_isolated_ref == 1 ): \n",
    "    preselection_df0 = preselection_df[preselection_df['isolated_citation']== string_isolated_ref ]  \n",
    "    \n",
    "    if string_isolated_ref ==0:\n",
    "        string_isolated_ref = \", group ref\"\n",
    "    elif string_isolated_ref ==1:\n",
    "        string_isolated_ref = \", isolated ref\"\n",
    "else:    \n",
    "    preselection_df0 = preselection_df   \n",
    "    print (\"size of preselection1 (by isolated/group ref):\",preselection_df0.shape, string_isolated_ref)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "######### preselection by plos ONE subject category:\n",
    "if string_code_categ==\"\": \n",
    "    preselection_df111 = preselection_df0\n",
    "else:    \n",
    "    if \" \" not in string_code_categ:  # to include one single category\n",
    "        preselection_df111 = preselection_df0[preselection_df0['categ_codes'].str.contains(string_code_categ)]        \n",
    "        string_code_categ = \" \"+dict_code_categ[string_code_categ]  \n",
    "        \n",
    "    else:  # if multiple codes-categories\n",
    "        list_codes = string_code_categ.split(\" \")\n",
    "        print (list_codes)\n",
    "\n",
    "        if len(list_codes) >= 2:              \n",
    "            preselection_df111 = preselection_df0[ preselection_df0['categ_codes'].str.contains('|'.join(list_codes)) ]  # to look for partial matches from a list of strings!!!!!\n",
    "            \n",
    "       \n",
    "        string_code_categ = \"\" \n",
    "        for code in list_codes:\n",
    "            string_code_categ += \"-\"+dict_code_categ[code] \n",
    "            \n",
    "            \n",
    "    print (\" size of preselection (by plos ONE subject category):\",preselection_df111.shape, string_code_categ)\n",
    "           \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "\n",
    "######### preselection by plos journal:\n",
    "if string_journal==\"\": \n",
    "    preselection_df1 = preselection_df111\n",
    "else:    \n",
    "    preselection_df1 = preselection_df111[preselection_df111['plos_j1']== string_journal ]  \n",
    "print (\" size of preselection2 (by plos journal):\",preselection_df1.shape, string_journal)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by plos field:\n",
    "if string_plos_field==\"\": \n",
    "    preselection_df2 = preselection_df1\n",
    "else:    \n",
    "    preselection_df2 = preselection_df1[preselection_df1['plos_field']== string_plos_field ]  \n",
    "print (\" size of preselection2 (by plos field):\",preselection_df2.shape, string_plos_field)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "preselection_df3 = preselection_df2\n",
    "   \n",
    "  \n",
    "\n",
    "\n",
    "\n",
    "if v1_string ==  'cite_count'  or       v1_string ==  'log_num_cit_ref'   or v1_string == 'log2_num_cit_ref':\n",
    "     \n",
    "      \n",
    "    \n",
    "    string_age_selection=''\n",
    "\n",
    "    ##### preselection only young/old references:        \n",
    "    if string_references_age == \"young\":\n",
    "        time_window = 1\n",
    "        string_age_selection=\"only young references from >=\"+ str((min(years)-time_window))\n",
    "        preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] >= (min(years)-time_window) ]   \n",
    "        print (\"  size of preselection3 (only young references):\",preselection_df3.shape, string_age_selection)\n",
    "        \n",
    "    elif string_references_age == \"old\":\n",
    "        time_window = 10\n",
    "        string_age_selection=\"only old references from <=\"+str((min(years)-time_window))\n",
    "        preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] <= (min(years)-time_window) ]   \n",
    "        print (\"  size of preselection3 (only young references):\",preselection_df3.shape,string_age_selection )\n",
    "        \n",
    "    else:\n",
    "        string_age_selection=\"young&old\"       \n",
    "        print (\"  No preselection by age of references:\",preselection_df3.shape )\n",
    "          \n",
    "    \n",
    "    \n",
    "\n",
    "N_plos=len(preselection_df3.paper_UT.unique())        ## this values are overall, for the title   \n",
    "N_all=len(preselection_df3)\n",
    " \n",
    "\n",
    "print (\"\\nTot # records included:\",len(preselection_df3),\"   # number of plos papers:\",len(preselection_df3.paper_UT.unique()), \"   # unique ref:\", len(preselection_df3.reference_UT.unique()),'\\n')\n",
    "\n",
    " \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "x1 = preselection_df3[v1_string]\n",
    "\n",
    "print (x1.value_counts())\n",
    "\n",
    "font_gral=60\n",
    "\n",
    "font=font_gral#20\n",
    "font_ticks=font_gral#70\n",
    "font_axes=font_gral#80\n",
    "\n",
    "\n",
    "trace1= Histogram(\n",
    "        x=x1, \n",
    "     marker=dict(\n",
    "        color='#9900ff',\n",
    "    ),\n",
    "       # name='high cit',\n",
    "       histnorm='probability',\n",
    "    #cumulative=dict(enabled=True)\n",
    ")\n",
    "\n",
    "\n",
    "layout = Layout(    \n",
    "\n",
    "    bargap=0.2,\n",
    "\n",
    "    xaxis=dict(\n",
    "        #title= 'Section' , #v1_string.replace(\"_\",\" \"),   \n",
    "        range=[-.49, 3.49],\n",
    "        titlefont=dict(            \n",
    "            size=font_axes,\n",
    "            color='black'\n",
    "        #    color='lightgrey'\n",
    "        ),  \n",
    "        tickfont=dict(              \n",
    "            size=font_ticks+30,\n",
    "            color='black'\n",
    "        ),\n",
    "        #type='log'\n",
    "        tickvals=[0,1,2,3],\n",
    "        #ticktext=[\"Introduction\",\"Methods\",\"Results\",\"Discussion\"]\n",
    "        ticktext=[\"I\",\"M\",\"R\",\"D\"]\n",
    "    ),\n",
    "    yaxis=dict(\n",
    "        title='Fraction of references',\n",
    "        #type='log',\n",
    "        titlefont=dict(\n",
    "            size=font_axes+30,\n",
    "            color='black'\n",
    "        #    color='lightgrey'\n",
    "        ),  \n",
    "        tickfont=dict(               \n",
    "            size=font_ticks+30,\n",
    "            color='black'\n",
    "        ),\n",
    "        \n",
    "    ),                \n",
    "\n",
    "    margin=go.Margin(\n",
    "        l=350,\n",
    "       # r=50,\n",
    "        b=150,\n",
    "        #t=100,\n",
    "       # pad=4\n",
    "    ),\n",
    "\n",
    "\n",
    "\n",
    ")       \n",
    "\n",
    "data = [trace1]\n",
    "\n",
    "fig = Figure(data=data, layout=layout)\n",
    "\n",
    "iplot(fig, filename='Fraction_references_each_section_overall.html')\n",
    "\n",
    "\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=\"fig1b\" ,\n",
    "             output_type='file', image_width=2000, image_height=1400, filename='/home/staff/julia/at_Northwestern/In_Text_Citations/In-Text-Citations-New/plots/fig1b.html', validate=False)\n",
    "\n",
    "\n",
    "print (\"occurrences:\", df_merged.shape, \"unique ref_UTs:\",len(df_merged.reference_UT.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}