examples/NLP_Examples/Example of Coword Mention Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "import scipy.sparse as spsparse\n",
    "\n",
    "import pyscisci.all as pyscisci\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the MAG database is stored\n",
    "path2mag = '/home/ajgates/MAG'\n",
    "path2mag = '/users/hgt6rn/Documents/DataSets/MAG'\n",
    "\n",
    "\n",
    "mymag = pyscisci.MAG(path2mag, database_extension='hdf', keep_in_memory=False, show_progress=True) \n",
    "# set keep_in_memory=False if you want to load the database each time its needed \n",
    "# otherwise keep_in_memory=True will keep each database in memory after its loaded\n",
    "\n",
    "# global_filter will only load publications that match the filtering criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will focus on a seminal paper that both measures and embodies interdisciplinarity:\n",
    "\n",
    "# A general framework for analysing diversity in science, technology and society\n",
    "# Andrew Stirling\n",
    "\n",
    "focus_publication_id = 2110158660\n",
    "\n",
    "\n",
    "# the idea here is to understand the interdisciplinary impact of the focus publication \n",
    "\n",
    "# to do this, we take all publications that cite the focus publication, and ask how these are related to eachother\n",
    "# by making the co-citation network\n",
    "# clusters/communities in this network reflect different home disciplines of the publications and \n",
    "# reflect the interdisciplinary nature of our original focus publication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9cf749cb86674fb2baf4fa429a7b3980",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a824fec7fe8a4ab2a136879e402feabf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11516 citing publications produce 427 co-cited publications\n"
     ]
    }
   ],
   "source": [
    "# First we load all of the publications citing our focus publication\n",
    "filter_dict = {'CitedPublicationId':np.sort([focus_publication_id])}\n",
    "citing_focus = mymag.load_references(filter_dict=filter_dict)\n",
    "\n",
    "focus_citing_pubs = np.sort(citing_focus['CitingPublicationId'].unique())\n",
    "\n",
    "# Then we need all of the publications that cite these publications\n",
    "pub2ref = mymag.load_references(filter_dict={'CitedPublicationId':focus_citing_pubs})\n",
    "\n",
    "print(\"{0} citing publications produce {1} co-cited publications\".format(pub2ref['CitingPublicationId'].nunique(), \n",
    "                                                                         pub2ref['CitedPublicationId'].nunique()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0712cc5551d4979a7a966897c156903",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publications:   0%|          | 0/132 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>FamilyId</th>\n",
       "      <th>Doi</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Issue</th>\n",
       "      <th>FirstPage</th>\n",
       "      <th>LastPage</th>\n",
       "      <th>DocSubTypes</th>\n",
       "      <th>DocType</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>351727</th>\n",
       "      <td>2356497335</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>138541872.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3743/KOSIM.2016.33.1.007</td>\n",
       "      <td>topic modeling based interdisciplinarity measu...</td>\n",
       "      <td>2016-03-30</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>32</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>731605</th>\n",
       "      <td>2921285431</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>148561398.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1007/S11192-019-03067-2</td>\n",
       "      <td>finding high impact interdisciplinary users ba...</td>\n",
       "      <td>2019-05-01</td>\n",
       "      <td>119</td>\n",
       "      <td>2</td>\n",
       "      <td>1017</td>\n",
       "      <td>1035</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1658733</th>\n",
       "      <td>2094373579</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1109/SOCIALCOM.2013.49</td>\n",
       "      <td>religious politicians and creative photographe...</td>\n",
       "      <td>2013-09-08</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>303</td>\n",
       "      <td>310</td>\n",
       "      <td></td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>769247</th>\n",
       "      <td>2086986353</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>39307421.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/J.TECHFORE.2012.12.005</td>\n",
       "      <td>social capital absorptive capability and firm ...</td>\n",
       "      <td>2013-09-01</td>\n",
       "      <td>80</td>\n",
       "      <td>7</td>\n",
       "      <td>1261</td>\n",
       "      <td>1270</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1236249</th>\n",
       "      <td>3101642834</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>196734849.0</td>\n",
       "      <td>3.101643e+09</td>\n",
       "      <td>10.1038/S41598-019-38869-0</td>\n",
       "      <td>assessing diversity in multiplex networks</td>\n",
       "      <td>2019-03-14</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4511</td>\n",
       "      <td>4511</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1073442</th>\n",
       "      <td>3123394243</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.123394e+09</td>\n",
       "      <td>10.1002/ASI.23631</td>\n",
       "      <td>strategic intelligence on emerging technologie...</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>233</td>\n",
       "      <td></td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1084441</th>\n",
       "      <td>2175567070</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>9731383.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/J.RESPOL.2015.06.014</td>\n",
       "      <td>internal or external spillovers which kind of ...</td>\n",
       "      <td>2016-02-01</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>41</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189762</th>\n",
       "      <td>1923918141</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>powering our lives sustainable energy manageme...</td>\n",
       "      <td>2008-01-01</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363717</th>\n",
       "      <td>2027366007</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>67638930.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/J.STRUECO.2008.10.001</td>\n",
       "      <td>multilevel assessment of diversity innovation ...</td>\n",
       "      <td>2009-03-01</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>60</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1009860</th>\n",
       "      <td>2752181393</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>58516903.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/J.JCLEPRO.2017.08.234</td>\n",
       "      <td>profiling research of the engineering academic...</td>\n",
       "      <td>2018-01-20</td>\n",
       "      <td>172</td>\n",
       "      <td></td>\n",
       "      <td>4239</td>\n",
       "      <td>4253</td>\n",
       "      <td></td>\n",
       "      <td>j</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>427 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         PublicationId    Year    JournalId      FamilyId  \\\n",
       "351727      2356497335  2016.0  138541872.0           NaN   \n",
       "731605      2921285431  2019.0  148561398.0           NaN   \n",
       "1658733     2094373579  2013.0          NaN           NaN   \n",
       "769247      2086986353  2013.0   39307421.0           NaN   \n",
       "1236249     3101642834  2019.0  196734849.0  3.101643e+09   \n",
       "...                ...     ...          ...           ...   \n",
       "1073442     3123394243  2017.0          NaN  3.123394e+09   \n",
       "1084441     2175567070  2016.0    9731383.0           NaN   \n",
       "189762      1923918141  2008.0          NaN           NaN   \n",
       "363717      2027366007  2009.0   67638930.0           NaN   \n",
       "1009860     2752181393  2018.0   58516903.0           NaN   \n",
       "\n",
       "                                    Doi  \\\n",
       "351727      10.3743/KOSIM.2016.33.1.007   \n",
       "731605       10.1007/S11192-019-03067-2   \n",
       "1658733       10.1109/SOCIALCOM.2013.49   \n",
       "769247   10.1016/J.TECHFORE.2012.12.005   \n",
       "1236249      10.1038/S41598-019-38869-0   \n",
       "...                                 ...   \n",
       "1073442               10.1002/ASI.23631   \n",
       "1084441    10.1016/J.RESPOL.2015.06.014   \n",
       "189762                                    \n",
       "363717    10.1016/J.STRUECO.2008.10.001   \n",
       "1009860   10.1016/J.JCLEPRO.2017.08.234   \n",
       "\n",
       "                                                     Title        Date Volume  \\\n",
       "351727   topic modeling based interdisciplinarity measu...  2016-03-30     33   \n",
       "731605   finding high impact interdisciplinary users ba...  2019-05-01    119   \n",
       "1658733  religious politicians and creative photographe...  2013-09-08          \n",
       "769247   social capital absorptive capability and firm ...  2013-09-01     80   \n",
       "1236249          assessing diversity in multiplex networks  2019-03-14      9   \n",
       "...                                                    ...         ...    ...   \n",
       "1073442  strategic intelligence on emerging technologie...  2017-01-01     68   \n",
       "1084441  internal or external spillovers which kind of ...  2016-02-01     45   \n",
       "189762   powering our lives sustainable energy manageme...  2008-01-01          \n",
       "363717   multilevel assessment of diversity innovation ...  2009-03-01     20   \n",
       "1009860  profiling research of the engineering academic...  2018-01-20    172   \n",
       "\n",
       "        Issue FirstPage LastPage DocSubTypes DocType  \n",
       "351727      1         7       32                   j  \n",
       "731605      2      1017     1035                   j  \n",
       "1658733             303      310                   c  \n",
       "769247      7      1261     1270                   j  \n",
       "1236249     1      4511     4511                   j  \n",
       "...       ...       ...      ...         ...     ...  \n",
       "1073442     1       214      233                   c  \n",
       "1084441     1        27       41                   j  \n",
       "189762                                                \n",
       "363717      1        50       60                   j  \n",
       "1009860            4239     4253                   j  \n",
       "\n",
       "[427 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now lets get the publication information\n",
    "pub = mymag.load_publications(filter_dict={'PublicationId':np.sort(pub2ref['CitedPublicationId'].unique())})\n",
    "pub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144 nodes, and 1494 edges\n"
     ]
    }
   ],
   "source": [
    "# we are going to make the co-citation network between all of the publications that cited our focus publication\n",
    "comentionnet, word2int = pyscisci.coword_network(pub, text_column='Title', stop_words= 'english', \n",
    "                                                strip_accents='ascii', lowercase=True, threshold=5, \n",
    "                                                vocabulary=None, show_progress=False)\n",
    "\n",
    "# elminate self-loops\n",
    "comentionnet.setdiag(0)\n",
    "\n",
    "comentionnet.data[comentionnet.data <=1] = 0\n",
    "comentionnet.eliminate_zeros()\n",
    "\n",
    "# comentionnet is our network adjcency matrix\n",
    "# word2int maps the words to the row/col of the adjcency matrix\n",
    "print(\"{0} nodes, and {1} edges\".format(len(word2int), comentionnet.nnz))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Local cdn resources have problems on chrome/safari when used in jupyter-notebook. \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"600px\"\n",
       "            src=\"CoWordMention.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2a3274100>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets make a fancy visualization using pyviz\n",
    "#from pyvis.network import Network\n",
    "\n",
    "g = nx.from_scipy_sparse_array(comentionnet, edge_attribute='weight')\n",
    "g = nx.relabel_nodes(g, {i:w for w,i in word2int.items()})\n",
    "nx.set_edge_attributes(g, {e:int(w) for e,w in nx.get_edge_attributes(g, 'weight').items()}, \"weight\")\n",
    "\n",
    "net = Network(notebook=True)\n",
    "net.repulsion()\n",
    "net.from_nx(g)\n",
    "net.show(\"CoWordMention.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}