src/Manuscript_Code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# %pylab inline\n",
    "from pylab import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_directory = '../data/'\n",
    "cmdstan_directory = 'cmdstan-2.9.0/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os import path\n",
    "import sys\n",
    "abs_path_data_directory = path.abspath(data_directory)+'/'\n",
    "python_location = path.dirname(sys.executable)+'/python'\n",
    "cmdstan_directory = path.abspath(cmdstan_directory)+'/'\n",
    "### Necessary for submitting analyses to a cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "def create_directory_if_not_existing(f):\n",
    "    try:\n",
    "        os.makedirs(f)\n",
    "    except OSError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Organize data for citations, co-classifications and occurrences\n",
    "==="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print(\"Organizing Citations\")\n",
    "# %run -i Organize_Citations.py\n",
    "# print(\"Organizing Classifications\")\n",
    "# %run -i Organize_Classifications.py\n",
    "# print(\"Organizing Occurrences\")\n",
    "# %run -i Organize_Occurrences.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define parameters\n",
    "==="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define classes and entities to analyze\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class_systems = ['IPC4']\n",
    "occurrence_entities = {#'Firm': ('occurrences_organized.h5', 'entity_classes_Firm'),\n",
    "                       'Inventor': ('occurrences_organized.h5', 'entity_classes_Inventor'),\n",
    "                       'PID': ('classifications_organized.h5', 'patent_classes'),\n",
    "                       }\n",
    "entity_types = list(occurrence_entities.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define what years to calculate networks for\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target_years = 'all'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define number of years of history networks should include\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_n_years = ['all', 1]\n",
    "\n",
    "def create_n_years_label(n_years):\n",
    "    if n_years is None or n_years=='all' or n_years=='cumulative':\n",
    "        n_years_label = ''\n",
    "    else:\n",
    "        n_years_label = '%i_years_'%n_years\n",
    "    return n_years_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate empirical networks\n",
    "==="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "citation_metrics = ['Class_Cites_Class_Count',\n",
    "                    'Class_Cited_by_Class_Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/')\n",
    "create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/citations/')\n",
    "create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/cooccurrence/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating for IPC4------\n",
      "Calculating for all years------\n",
      "1976\n",
      "0.00 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.02 sec\n",
      "0.02 sec\n",
      "1977\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.03 sec\n",
      "0.03 sec\n",
      "1978\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.04 sec\n",
      "0.04 sec\n",
      "1979\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.06 sec\n",
      "0.06 sec\n",
      "1980\n",
      "0.03 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.09 sec\n",
      "0.09 sec\n",
      "1981\n",
      "0.04 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.14 sec\n",
      "0.14 sec\n",
      "1982\n",
      "0.05 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.18 sec\n",
      "0.19 sec\n",
      "1983\n",
      "0.07 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.23 sec\n",
      "0.24 sec\n",
      "1984\n",
      "0.08 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.29 sec\n",
      "0.31 sec\n",
      "1985\n",
      "0.11 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.36 sec\n",
      "0.39 sec\n",
      "1986\n",
      "0.13 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.46 sec\n",
      "0.48 sec\n",
      "1987\n",
      "0.18 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.56 sec\n",
      "0.61 sec\n",
      "1988\n",
      "0.20 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.66 sec\n",
      "0.71 sec\n",
      "1989\n",
      "0.25 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.81 sec\n",
      "0.85 sec\n",
      "1990\n",
      "0.30 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.93 sec\n",
      "1.01 sec\n",
      "1991\n",
      "0.35 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.08 sec\n",
      "1.20 sec\n",
      "1992\n",
      "0.41 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.37 sec\n",
      "1.47 sec\n",
      "1993\n",
      "0.46 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.56 sec\n",
      "1.75 sec\n",
      "1994\n",
      "0.55 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.72 sec\n",
      "1.99 sec\n",
      "1995\n",
      "0.61 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.94 sec\n",
      "2.25 sec\n",
      "1996\n",
      "0.69 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "2.24 sec\n",
      "2.54 sec\n",
      "1997\n",
      "0.82 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "2.43 sec\n",
      "2.90 sec\n",
      "1998\n",
      "0.92 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.02 sec\n",
      "3.64 sec\n",
      "1999\n",
      "1.05 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.33 sec\n",
      "4.04 sec\n",
      "2000\n",
      "1.24 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.76 sec\n",
      "4.65 sec\n",
      "2001\n",
      "1.45 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "4.18 sec\n",
      "5.23 sec\n",
      "2002\n",
      "1.64 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "4.67 sec\n",
      "5.97 sec\n",
      "2003\n",
      "1.81 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "5.24 sec\n",
      "6.82 sec\n",
      "2004\n",
      "2.03 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "5.72 sec\n",
      "7.96 sec\n",
      "2005\n",
      "2.34 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "6.76 sec\n",
      "8.69 sec\n",
      "2006\n",
      "2.65 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "7.42 sec\n",
      "9.56 sec\n",
      "2007\n",
      "2.88 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "7.91 sec\n",
      "10.70 sec\n",
      "2008\n",
      "3.12 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "8.50 sec\n",
      "11.76 sec\n",
      "2009\n",
      "3.38 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "9.07 sec\n",
      "12.54 sec\n",
      "2010\n",
      "3.89 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "9.90 sec\n",
      "13.94 sec\n",
      "Time until calculations are done: 321.09\n",
      "Total runtime: 360.17\n",
      "PID\n",
      "Inventor\n",
      "Calculating for IPC4------\n",
      "Calculating for 1 years------\n",
      "1976\n",
      "0.00 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.02 sec\n",
      "0.02 sec\n",
      "1977\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.02 sec\n",
      "0.02 sec\n",
      "1978\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.04 sec\n",
      "0.03 sec\n",
      "1979\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.04 sec\n",
      "0.03 sec\n",
      "1980\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.06 sec\n",
      "0.05 sec\n",
      "1981\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.07 sec\n",
      "0.06 sec\n",
      "1982\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.08 sec\n",
      "0.06 sec\n",
      "1983\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.08 sec\n",
      "0.06 sec\n",
      "1984\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.10 sec\n",
      "0.08 sec\n",
      "1985\n",
      "0.03 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.13 sec\n",
      "0.09 sec\n",
      "1986\n",
      "0.03 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.13 sec\n",
      "0.09 sec\n",
      "1987\n",
      "0.04 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.17 sec\n",
      "0.11 sec\n",
      "1988\n",
      "0.04 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.18 sec\n",
      "0.11 sec\n",
      "1989\n",
      "0.05 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.23 sec\n",
      "0.14 sec\n",
      "1990\n",
      "0.05 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.23 sec\n",
      "0.14 sec\n",
      "1991\n",
      "0.06 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.26 sec\n",
      "0.15 sec\n",
      "1992\n",
      "0.06 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.30 sec\n",
      "0.17 sec\n",
      "1993\n",
      "0.07 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.32 sec\n",
      "0.17 sec\n",
      "1994\n",
      "0.08 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.36 sec\n",
      "0.19 sec\n",
      "1995\n",
      "0.08 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.39 sec\n",
      "0.20 sec\n",
      "1996\n",
      "0.09 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.44 sec\n",
      "0.23 sec\n",
      "1997\n",
      "0.10 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.47 sec\n",
      "0.24 sec\n",
      "1998\n",
      "0.14 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.63 sec\n",
      "0.33 sec\n",
      "1999\n",
      "0.15 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.69 sec\n",
      "0.35 sec\n",
      "2000\n",
      "0.17 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.75 sec\n",
      "0.38 sec\n",
      "2001\n",
      "0.18 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.83 sec\n",
      "0.41 sec\n",
      "2002\n",
      "0.21 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.90 sec\n",
      "0.45 sec\n",
      "2003\n",
      "0.22 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.98 sec\n",
      "0.49 sec\n",
      "2004\n",
      "0.23 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.98 sec\n",
      "0.49 sec\n",
      "2005\n",
      "0.22 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.94 sec\n",
      "0.44 sec\n",
      "2006\n",
      "0.29 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.15 sec\n",
      "0.59 sec\n",
      "2007\n",
      "0.26 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.10 sec\n",
      "0.54 sec\n",
      "2008\n",
      "0.27 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.12 sec\n",
      "0.54 sec\n",
      "2009\n",
      "0.31 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.30 sec\n",
      "0.58 sec\n",
      "2010\n",
      "0.43 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.65 sec\n",
      "0.81 sec\n",
      "Time until calculations are done: 49.79\n",
      "Total runtime: 87.12\n",
      "PID\n",
      "Inventor\n",
      "Calculating for IPC4------\n",
      "Calculating for 5 years------\n",
      "1976\n",
      "0.00 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.02 sec\n",
      "0.02 sec\n",
      "1977\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.02 sec\n",
      "0.02 sec\n",
      "1978\n",
      "0.01 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.04 sec\n",
      "0.04 sec\n",
      "1979\n",
      "0.02 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.06 sec\n",
      "0.06 sec\n",
      "1980\n",
      "0.03 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.09 sec\n",
      "0.09 sec\n",
      "1981\n",
      "0.04 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.13 sec\n",
      "0.14 sec\n",
      "1982\n",
      "0.05 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.17 sec\n",
      "0.18 sec\n",
      "1983\n",
      "0.06 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.21 sec\n",
      "0.20 sec\n",
      "1984\n",
      "0.07 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.26 sec\n",
      "0.25 sec\n",
      "1985\n",
      "0.09 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.31 sec\n",
      "0.29 sec\n",
      "1986\n",
      "0.10 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.36 sec\n",
      "0.32 sec\n",
      "1987\n",
      "0.12 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.45 sec\n",
      "0.40 sec\n",
      "1988\n",
      "0.14 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.51 sec\n",
      "0.44 sec\n",
      "1989\n",
      "0.18 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.61 sec\n",
      "0.53 sec\n",
      "1990\n",
      "0.19 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.69 sec\n",
      "0.58 sec\n",
      "1991\n",
      "0.21 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.77 sec\n",
      "0.66 sec\n",
      "1992\n",
      "0.24 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.86 sec\n",
      "0.73 sec\n",
      "1993\n",
      "0.27 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "0.96 sec\n",
      "0.79 sec\n",
      "1994\n",
      "0.30 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.04 sec\n",
      "0.84 sec\n",
      "1995\n",
      "0.32 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.14 sec\n",
      "0.92 sec\n",
      "1996\n",
      "0.36 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.34 sec\n",
      "0.99 sec\n",
      "1997\n",
      "0.40 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.47 sec\n",
      "1.07 sec\n",
      "1998\n",
      "0.47 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.66 sec\n",
      "1.33 sec\n",
      "1999\n",
      "0.56 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "1.93 sec\n",
      "1.78 sec\n",
      "2000\n",
      "0.65 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "2.18 sec\n",
      "1.82 sec\n",
      "2001\n",
      "0.72 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "2.41 sec\n",
      "2.30 sec\n",
      "2002\n",
      "0.83 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "2.92 sec\n",
      "2.57 sec\n",
      "2003\n",
      "0.89 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.16 sec\n",
      "2.55 sec\n",
      "2004\n",
      "0.97 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.22 sec\n",
      "2.73 sec\n",
      "2005\n",
      "0.98 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.43 sec\n",
      "3.27 sec\n",
      "2006\n",
      "1.09 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.75 sec\n",
      "3.17 sec\n",
      "2007\n",
      "1.14 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.85 sec\n",
      "3.35 sec\n",
      "2008\n",
      "1.21 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "3.96 sec\n",
      "3.35 sec\n",
      "2009\n",
      "1.33 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "4.10 sec\n",
      "3.45 sec\n",
      "2010\n",
      "1.53 sec\n",
      "Class_Cites_Class_Count\n",
      "Class_Cited_by_Class_Count\n",
      "4.58 sec\n",
      "4.13 sec\n",
      "Time until calculations are done: 138.71\n",
      "Total runtime: 177.79\n",
      "PID\n",
      "Inventor\n"
     ]
    }
   ],
   "source": [
    "# ### Create empirical networks\n",
    "# randomized_control = False\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     for n_years in all_n_years:\n",
    "#         print(\"Calculating for %s------\"%class_system)\n",
    "#         print(\"Calculating for %s years------\"%str(n_years))\n",
    "#         ### Calculate citation networks\n",
    "#         %run -i Calculating_Citation_Networks.py\n",
    "#         all_networks = networks\n",
    "\n",
    "#         ### Calculate co-occurrence networks\n",
    "#         preverse_years = True\n",
    "#         for entity_column in entity_types:\n",
    "#             target_years = 'all'\n",
    "#             print(entity_column)\n",
    "#             occurrence_data, entity_data = occurrence_entities[entity_column]\n",
    "#             %run -i Calculating_CoOccurrence_Networks.py\n",
    "#             all_networks.ix['Class_CoOccurrence_Count_%s'%entity_column] = networks\n",
    "\n",
    "#         ind = ['Class_CoOccurrence_Count_%s'%entity for entity in entity_types]\n",
    "#         store = pd.HDFStore(data_directory+'Class_Relatedness_Networks/cooccurrence/class_relatedness_networks_cooccurrence.h5', \n",
    "#                         mode='a', table=True)\n",
    "#         n_years_label = create_n_years_label(n_years)\n",
    "#         store.put('/empirical_cooccurrence_%s%s'%(n_years_label,class_system), all_networks.ix[ind], 'table', append=False)\n",
    "#         store.close()\n",
    "\n",
    "#         #### Combine them both\n",
    "#         store = pd.HDFStore(data_directory+'Class_Relatedness_Networks/class_relatedness_networks.h5', \n",
    "#                             mode='a', table=True)\n",
    "#         store.put('/empirical_'+n_years_label+class_system, all_networks, 'table', append=False)\n",
    "#         store.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate randomized, synthetic networks\n",
    "===="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make directories\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/citations/controls/')\n",
    "create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/cooccurrence/controls/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run randomizations\n",
    "---\n",
    "(Currently set up to use a cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "first_rand_id = 0\n",
    "n_randomizations = 1000\n",
    "overwrite = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create_directory_if_not_existing('jobfiles/')\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     for n_years in all_n_years:\n",
    "#         ### Citations\n",
    "#         create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/citations/controls/'+class_system)\n",
    "#         basic_program = open('Calculating_Citation_Networks.py', 'r').read()\n",
    "#         job_type = 'citations'\n",
    "#         options=\"\"\"class_system = %r\n",
    "# target_years = %r\n",
    "# n_years = %r\n",
    "# data_directory = %r\n",
    "# randomized_control = True\n",
    "# citation_metrics = %r\n",
    "#     \"\"\"%(class_system, target_years, n_years, abs_path_data_directory, citation_metrics)\n",
    "\n",
    "#         %run -i Calculating_Synthetic_Networks_Control_Commands\n",
    "\n",
    "#         ### Co-occurrences\n",
    "#         create_directory_if_not_existing(data_directory+'Class_Relatedness_Networks/cooccurrence/controls/'+class_system)\n",
    "#         basic_program = open('Calculating_CoOccurrence_Networks.py', 'r').read()\n",
    "#         job_type = 'cooccurrence'\n",
    "#         for entity in entity_types:\n",
    "#             occurrence_data, entity_data = occurrence_entities[entity]\n",
    "#             options = \"\"\"class_system = %r\n",
    "# target_years = %r\n",
    "# n_years = %r\n",
    "# data_directory = %r\n",
    "# randomized_control = True\n",
    "# preserve_years = True\n",
    "# chain = False\n",
    "# occurrence_data = %r\n",
    "# entity_data = %r\n",
    "# entity_column = %r\n",
    "# print(occurrence_data)\n",
    "# print(entity_data)\n",
    "# print(entity_column)\n",
    "#     \"\"\"%(class_system, target_years, n_years, abs_path_data_directory, occurrence_data, entity_data, entity)\n",
    "\n",
    "#             %run -i Calculating_Synthetic_Networks_Control_Commands"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrate randomized data and calculate Z-scores\n",
    "---\n",
    "Note: Any classes that have no data (i.e. no patents within that class) will create z-scores of 'nan', which will be dropped when saved to the HDF5 file. Therefore, the z-scores data will simply not includes these classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IPC4\n",
      "all\n",
      "\n",
      "Combining Outputs\n",
      "1\n",
      "1_years_\n",
      "Combining Outputs\n",
      "5\n",
      "5_years_\n",
      "Combining Outputs\n"
     ]
    }
   ],
   "source": [
    "# n_controls = n_randomizations\n",
    "\n",
    "# output_citations = 'class_relatedness_networks_citations'\n",
    "# # output_citations = False\n",
    "# output_cooccurrence = 'class_relatedness_networks_cooccurrence'\n",
    "# # output_cooccurrence = False\n",
    "# combine_outputs = True\n",
    "\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     print(class_system)\n",
    "#     for n_years in all_n_years:\n",
    "#         print(n_years)\n",
    "#         n_years_label = create_n_years_label(n_years)\n",
    "#         cooccurrence_base_file_name = 'synthetic_control_cooccurrence_'+n_years_label+'%s_preserve_years_%s'\n",
    "\n",
    "#         %run -i Calculating_Synthetic_Networks_Integrate_Runs.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete individual runs of randomized data\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from shutil import rmtree\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     rmtree(data_directory+'Class_Relatedness_Networks/citations/controls/'+class_system)\n",
    "#     rmtree(data_directory+'Class_Relatedness_Networks/cooccurrence/controls/'+class_system)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regress out popularity from relatedness measures\n",
    "---\n",
    "First create popularity-by-year networks for all class systems and n_years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# %run -i Calculating_Popularity_Networks.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %run -i Regressing_Popularity_Out_of_Z_Scores.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create inventor entries data\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Precalculate some data that all runs will rely on and which takes a long time to calculate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_length = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for class_system in class_systems:\n",
    "#     %run -i Organize_Inventor_Patent_Data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for class_system in class_systems:\n",
    "#     %run -i Calculating_Supporting_Data_for_Agent_Entry_Calculations.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# overwrite = False\n",
    "\n",
    "# create_directory_if_not_existing('jobfiles/')\n",
    "# create_directory_if_not_existing(data_directory+'Agent_Entries/')\n",
    "# create_directory_if_not_existing(data_directory+'Agent_Entries/samples/')\n",
    "\n",
    "# for class_system in class_systems:\n",
    "\n",
    "#     store = pd.HDFStore(data_directory+'organized_patent_data.h5')\n",
    "#     agents_lookup_explorers = store['agents_lookup_explorers_%s'%class_system]\n",
    "#     store.close()\n",
    "\n",
    "#     n_agents = agents_lookup_explorers.shape[0]\n",
    "\n",
    "\n",
    "#     all_samples_end = n_agents+sample_length\n",
    "#     all_samples_start = 0\n",
    "#     %run -i Calculating_Inventor_Entries_Data_Control_Commands.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict inventor entries\n",
    "==="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "relatedness_types = [ 'Class_Cites_Class_Count_1_years_percent_positive_all_years_back_mean',\n",
    "                     'Class_Cited_by_Class_Count_1_years_percent_positive_all_years_back_mean',\n",
    "                     'Class_CoOccurrence_Count_Inventor_1_years_percent_positive_all_years_back_mean',\n",
    "                     'Class_CoOccurrence_Count_PID_1_years_percent_positive_all_years_back_mean',\n",
    "                    ]\n",
    "\n",
    "rescale_z = True #Doesn't actually matter if you're not using z-score based R, but it is necessary to assign"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate PDFs for each sample of inventors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create_directory_if_not_existing('jobfiles/')\n",
    "\n",
    "# for class_system in class_systems:\n",
    "\n",
    "#     store = pd.HDFStore(data_directory+'organized_patent_data.h5')\n",
    "#     agents_lookup_explorers = store['agents_lookup_explorers_%s'%class_system]\n",
    "#     store.close()\n",
    "\n",
    "#     n_agents = agents_lookup_explorers.shape[0]\n",
    "\n",
    "#     all_samples_end = n_agents#+sample_length\n",
    "#     all_samples_start = 0\n",
    "#     samples = range(all_samples_start,all_samples_end,sample_length)\n",
    "    \n",
    "#     rescale_z = True\n",
    "#     %run -i Fit_PDF_4D_to_Samples_Control_Commands.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrate the PDFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create_directory_if_not_existing('jobfiles/')\n",
    "# for class_system in class_systems:\n",
    "\n",
    "#     store = pd.HDFStore(data_directory+'organized_patent_data.h5')\n",
    "#     agents_lookup_explorers = store['agents_lookup_explorers_%s'%class_system]\n",
    "#     store.close()\n",
    "\n",
    "#     n_agents = agents_lookup_explorers.shape[0]\n",
    "\n",
    "#     max_sample = n_agents\n",
    "#     sample_start = 0\n",
    "#     n_combine = 100000\n",
    "\n",
    "#     %run -i Fit_PDF_4D_to_Samples_Integrate_Control_Commands.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the PDFs to predict the samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create_directory_if_not_existing('jobfiles/')\n",
    "# for class_system in class_systems:\n",
    "\n",
    "#     store = pd.HDFStore(data_directory+'organized_patent_data.h5')\n",
    "#     agents_lookup_explorers = store['agents_lookup_explorers_%s'%class_system]\n",
    "#     store.close()\n",
    "\n",
    "#     n_agents = agents_lookup_explorers.shape[0]\n",
    "#     all_samples_end = n_agents#+sample_length\n",
    "#     all_samples_start = 0\n",
    "#     samples = range(all_samples_start,all_samples_end,sample_length)\n",
    "\n",
    "#     %run -i Predict_Samples_NB_4D_Control_Commands.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrate the data on entries with predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# target = 'entries_with_predictions_NB_4D'\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     %run -i Calculating_Inventor_Entries_Data_Integrate_Runs.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performance modeling\n",
    "===="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = 'entries'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate inventors' performance after entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for class_system in class_systems:\n",
    "#     %run -i Calculating_Performance_Data.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create and compile performance models with Stan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %run -i Stan_Models.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use HMC sampling to do Bayesian inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# n_chains = 50\n",
    "# chains_start = 0\n",
    "# num_warmup = 300\n",
    "# num_samples = 300\n",
    "# n_observations = 'full'#10000\n",
    "\n",
    "# relatedness_type = 'Class_Cited_by_Class_Count'\n",
    "# relatedness = '%s_1_years_percent_positive_all_years_back_mean'%relatedness_type\n",
    "# popularity = 'Class_Patent_Count_1_years_Previous_Year_Percentile'\n",
    "\n",
    "# count_variables = ['Agent_Number_of_Patents_in_Class','Agent_Number_of_Citations_in_Class']\n",
    "# hit_thresholds = [3,4,5,6]\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     %run -i Stan_Sampling.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrate the samples together "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run -i Stan_Sampling_Integrate.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make figures\n",
    "==="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# figures_directory = '../manuscript/figs/'\n",
    "# save_as_manuscript_figures = True\n",
    "\n",
    "# for class_system in class_systems:\n",
    "#     %run -i Manuscript_Figures.py"
   ]
  }
 ],
 "metadata": {
  "css": [
   ""
  ],
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}