ChinaMigrants.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ff98c270-0756-480a-9b1f-c6d1994df46e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "29ba1c53",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SetEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, set):\n",
    "            return list(obj)\n",
    "        return json.JSONEncoder.default(self, obj)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5c11d91b-44c8-4991-aaa6-fdd9a9016be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = pd.read_csv(\"data/Heard_Relational_Company_IDs.csv\")\n",
    "locations = pd.read_csv(\"data/Heard_Relational_Location.csv\")\n",
    "personinfos = pd.read_csv(\"data/Nodes_researchIT.csv\")\n",
    "edges = pd.read_csv(\"data/Edges_researchIT.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "78848577",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Label</th>\n",
       "      <th>Surname</th>\n",
       "      <th>GivenName</th>\n",
       "      <th>Suffix</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Nationality</th>\n",
       "      <th>Employer</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Location</th>\n",
       "      <th>ValidFrom</th>\n",
       "      <th>ValidTo</th>\n",
       "      <th>Source</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>Heard</td>\n",
       "      <td>Augustine</td>\n",
       "      <td>Sr</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>1845</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>ID0000006</td>\n",
       "      <td>Dixwell</td>\n",
       "      <td>George B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000002;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>ID0000002</td>\n",
       "      <td>Heard</td>\n",
       "      <td>John</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1845</td>\n",
       "      <td>1850</td>\n",
       "      <td>PS000001;PS000003;PS000004;PS000005</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>ID0000008</td>\n",
       "      <td>Roberts</td>\n",
       "      <td>Oliver E</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1845</td>\n",
       "      <td>1846</td>\n",
       "      <td>CS137007;PS000001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>ID0000008</td>\n",
       "      <td>Roberts</td>\n",
       "      <td>Oliver E</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0003</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>1847</td>\n",
       "      <td>1850</td>\n",
       "      <td>CS137007;PS000005</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>268</td>\n",
       "      <td>ID0000102</td>\n",
       "      <td>Farley</td>\n",
       "      <td>Gustavus</td>\n",
       "      <td>Jr</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Agent</td>\n",
       "      <td>LC000015</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>269</td>\n",
       "      <td>ID0000101</td>\n",
       "      <td>Fraser</td>\n",
       "      <td>John Alexander</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>British</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000015</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>270</td>\n",
       "      <td>ID0000124</td>\n",
       "      <td>Low</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Agent</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>'F'?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>271</td>\n",
       "      <td>ID0000126</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>H</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>272</td>\n",
       "      <td>ID0000133</td>\n",
       "      <td>Dorr</td>\n",
       "      <td>Eben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Male</td>\n",
       "      <td>American</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Agent</td>\n",
       "      <td>LC000015</td>\n",
       "      <td>1868</td>\n",
       "      <td>1868</td>\n",
       "      <td>SS000004</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ID      Label     Surname       GivenName Suffix Gender Nationality  \\\n",
       "0      2  ID0000001       Heard       Augustine     Sr   Male    American   \n",
       "1      4  ID0000006     Dixwell        George B    NaN   Male    American   \n",
       "2      6  ID0000002       Heard            John    NaN   Male    American   \n",
       "3      7  ID0000008     Roberts        Oliver E    NaN   Male    American   \n",
       "4      8  ID0000008     Roberts        Oliver E    NaN   Male    American   \n",
       "..   ...        ...         ...             ...    ...    ...         ...   \n",
       "254  268  ID0000102      Farley        Gustavus     Jr   Male    American   \n",
       "255  269  ID0000101      Fraser  John Alexander    NaN   Male     British   \n",
       "256  270  ID0000124         Low               O    NaN   Male         NaN   \n",
       "257  271  ID0000126  Livingston               H    NaN   Male    American   \n",
       "258  272  ID0000133        Dorr            Eben    NaN   Male    American   \n",
       "\n",
       "    Employer Occupation  Location  ValidFrom  ValidTo  \\\n",
       "0     CO0001    Partner  LC000001       1845     1847   \n",
       "1     CO0001    Partner  LC000003       1842     1847   \n",
       "2     CO0001    Partner  LC000003       1845     1850   \n",
       "3     CO0001      Clerk  LC000003       1845     1846   \n",
       "4     CO0003      Clerk  LC000002       1847     1850   \n",
       "..       ...        ...       ...        ...      ...   \n",
       "254   CO0020      Agent  LC000015       1876     1876   \n",
       "255   CO0020      Clerk  LC000015       1876     1876   \n",
       "256   CO0020      Agent  LC000017       1876     1876   \n",
       "257   CO0020      Clerk  LC000017       1876     1876   \n",
       "258   CO0001      Agent  LC000015       1868     1868   \n",
       "\n",
       "                                  Source Notes  \n",
       "0                      PS000001;PS000003   NaN  \n",
       "1             PS000001;PS000002;PS000003   NaN  \n",
       "2    PS000001;PS000003;PS000004;PS000005   NaN  \n",
       "3                      CS137007;PS000001   NaN  \n",
       "4                      CS137007;PS000005   NaN  \n",
       "..                                   ...   ...  \n",
       "254                             PS000020   NaN  \n",
       "255                             PS000020   NaN  \n",
       "256                             PS000020  'F'?  \n",
       "257                             PS000020   NaN  \n",
       "258                             SS000004   NaN  \n",
       "\n",
       "[259 rows x 14 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# personinfos\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6d0ce119",
   "metadata": {},
   "outputs": [],
   "source": [
    "companies.CompanyName.fillna('', inplace=True)\n",
    "companies.DateFounded.fillna('', inplace=True)\n",
    "companies.DateClosed.fillna('', inplace=True)\n",
    "companies.NationalAffiliation.fillna('', inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4efb0b33",
   "metadata": {},
   "outputs": [],
   "source": [
    "person = personinfos[['Label', 'Surname', 'GivenName', 'Suffix',\n",
    "                      'Gender', 'Nationality', 'Location', 'Employer']].copy()\n",
    "person = person.drop_duplicates()\n",
    "person = person.sort_values(by=['Label'])\n",
    "person.Suffix.fillna('', inplace=True)\n",
    "person.Nationality.fillna('', inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ccdd1385",
   "metadata": {},
   "outputs": [],
   "source": [
    "jobs = personinfos[['ID', 'Label', 'Employer', 'Occupation',\n",
    "                    'Location', 'ValidFrom', 'ValidTo', 'Source', 'Notes']].copy()\n",
    "jobs.ValidTo.fillna(jobs.ValidFrom, inplace=True)\n",
    "jobs.Occupation.fillna('', inplace=True)\n",
    "# jobs.Employer.fillna('CO0000', inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "294cbcc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_locations = pd.merge(jobs, locations, how='inner',\n",
    "                         left_on='Location', right_on='Loc_ID')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "38258173",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_x</th>\n",
       "      <th>Label</th>\n",
       "      <th>Employer</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Location_x</th>\n",
       "      <th>ValidFrom</th>\n",
       "      <th>ValidTo</th>\n",
       "      <th>Source</th>\n",
       "      <th>Notes</th>\n",
       "      <th>Loc_ID</th>\n",
       "      <th>Location_y</th>\n",
       "      <th>Lat</th>\n",
       "      <th>Lng</th>\n",
       "      <th>ID_y</th>\n",
       "      <th>CompanyName</th>\n",
       "      <th>NationalAffiliation</th>\n",
       "      <th>DateFounded</th>\n",
       "      <th>DateClosed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>1845</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>Boston</td>\n",
       "      <td>42.360278</td>\n",
       "      <td>-71.057778</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1844</td>\n",
       "      <td>PS000002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>Guangzhou</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>46</td>\n",
       "      <td>ID0000002</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>1859</td>\n",
       "      <td>1862</td>\n",
       "      <td>PS000007;PS000008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>22.300000</td>\n",
       "      <td>114.200000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>ID0000002</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1844</td>\n",
       "      <td>PS000002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>Guangzhou</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>256</td>\n",
       "      <td>ID0000002</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>22.300000</td>\n",
       "      <td>114.200000</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Heard &amp; Co</td>\n",
       "      <td>USA</td>\n",
       "      <td>1876.0</td>\n",
       "      <td>1876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>250</td>\n",
       "      <td>ID0000130</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1874</td>\n",
       "      <td>1874</td>\n",
       "      <td>PS000018</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>Kobe</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>262</td>\n",
       "      <td>ID0000131</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Agent</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>Guangzhou</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Heard &amp; Co</td>\n",
       "      <td>USA</td>\n",
       "      <td>1876.0</td>\n",
       "      <td>1876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>251</td>\n",
       "      <td>ID0000131</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>1875</td>\n",
       "      <td>1875</td>\n",
       "      <td>CS99217;PS000019</td>\n",
       "      <td>Son of Charles Fearon</td>\n",
       "      <td>LC000002</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>22.300000</td>\n",
       "      <td>114.200000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>254</td>\n",
       "      <td>ID0000132</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1875</td>\n",
       "      <td>1875</td>\n",
       "      <td>PS000019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>Kobe</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>272</td>\n",
       "      <td>ID0000133</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Agent</td>\n",
       "      <td>LC000015</td>\n",
       "      <td>1868</td>\n",
       "      <td>1868</td>\n",
       "      <td>SS000004</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000015</td>\n",
       "      <td>Yokohama</td>\n",
       "      <td>35.444167</td>\n",
       "      <td>139.638056</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ID_x      Label Employer Occupation Location_x  ValidFrom  ValidTo  \\\n",
       "0       2  ID0000001   CO0001    Partner   LC000001       1845     1847   \n",
       "4      11  ID0000001   CO0001    Partner   LC000003       1842     1844   \n",
       "29     46  ID0000002   CO0001    Partner   LC000002       1859     1862   \n",
       "5      12  ID0000002   CO0001      Clerk   LC000003       1842     1844   \n",
       "223   256  ID0000002   CO0020    Partner   LC000002       1876     1876   \n",
       "..    ...        ...      ...        ...        ...        ...      ...   \n",
       "206   250  ID0000130   CO0001      Clerk   LC000017       1874     1874   \n",
       "222   262  ID0000131   CO0020      Agent   LC000003       1876     1876   \n",
       "89    251  ID0000131   CO0001      Clerk   LC000002       1875     1875   \n",
       "207   254  ID0000132   CO0001      Clerk   LC000017       1875     1875   \n",
       "199   272  ID0000133   CO0001      Agent   LC000015       1868     1868   \n",
       "\n",
       "                Source                  Notes    Loc_ID Location_y        Lat  \\\n",
       "0    PS000001;PS000003                    NaN  LC000001     Boston  42.360278   \n",
       "4             PS000002                    NaN  LC000003  Guangzhou  23.130000   \n",
       "29   PS000007;PS000008                    NaN  LC000002  Hong Kong  22.300000   \n",
       "5             PS000002                    NaN  LC000003  Guangzhou  23.130000   \n",
       "223           PS000020                    NaN  LC000002  Hong Kong  22.300000   \n",
       "..                 ...                    ...       ...        ...        ...   \n",
       "206           PS000018                    NaN  LC000017       Kobe  34.690000   \n",
       "222           PS000020                    NaN  LC000003  Guangzhou  23.130000   \n",
       "89    CS99217;PS000019  Son of Charles Fearon  LC000002  Hong Kong  22.300000   \n",
       "207           PS000019                    NaN  LC000017       Kobe  34.690000   \n",
       "199           SS000004                    NaN  LC000015   Yokohama  35.444167   \n",
       "\n",
       "            Lng    ID_y                CompanyName NationalAffiliation  \\\n",
       "0    -71.057778  CO0001  Augustine Heard & Company                 USA   \n",
       "4    113.260000  CO0001  Augustine Heard & Company                 USA   \n",
       "29   114.200000  CO0001  Augustine Heard & Company                 USA   \n",
       "5    113.260000  CO0001  Augustine Heard & Company                 USA   \n",
       "223  114.200000  CO0020                 Heard & Co                 USA   \n",
       "..          ...     ...                        ...                 ...   \n",
       "206  135.195556  CO0001  Augustine Heard & Company                 USA   \n",
       "222  113.260000  CO0020                 Heard & Co                 USA   \n",
       "89   114.200000  CO0001  Augustine Heard & Company                 USA   \n",
       "207  135.195556  CO0001  Augustine Heard & Company                 USA   \n",
       "199  139.638056  CO0001  Augustine Heard & Company                 USA   \n",
       "\n",
       "    DateFounded DateClosed  \n",
       "0        1840.0       1875  \n",
       "4        1840.0       1875  \n",
       "29       1840.0       1875  \n",
       "5        1840.0       1875  \n",
       "223      1876.0       1876  \n",
       "..          ...        ...  \n",
       "206      1840.0       1875  \n",
       "222      1876.0       1876  \n",
       "89       1840.0       1875  \n",
       "207      1840.0       1875  \n",
       "199      1840.0       1875  \n",
       "\n",
       "[259 rows x 18 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_locations_companies = pd.merge(\n",
    "    job_locations, companies, how='inner', left_on='Employer', right_on='ID')\n",
    "job_locations_companies.sort_values(by=['Label'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "124d7783",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_x</th>\n",
       "      <th>Label</th>\n",
       "      <th>Employer</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Location_x</th>\n",
       "      <th>ValidFrom</th>\n",
       "      <th>ValidTo</th>\n",
       "      <th>Source</th>\n",
       "      <th>Notes</th>\n",
       "      <th>Loc_ID</th>\n",
       "      <th>...</th>\n",
       "      <th>Lat</th>\n",
       "      <th>Lng</th>\n",
       "      <th>ID_y</th>\n",
       "      <th>CompanyName</th>\n",
       "      <th>NationalAffiliation</th>\n",
       "      <th>DateFounded</th>\n",
       "      <th>DateClosed</th>\n",
       "      <th>Surname</th>\n",
       "      <th>GivenName</th>\n",
       "      <th>Suffix</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>1845</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>...</td>\n",
       "      <td>42.360278</td>\n",
       "      <td>-71.057778</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Heard</td>\n",
       "      <td>Augustine</td>\n",
       "      <td>Sr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>1845</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000001</td>\n",
       "      <td>...</td>\n",
       "      <td>42.360278</td>\n",
       "      <td>-71.057778</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Heard</td>\n",
       "      <td>Augustine</td>\n",
       "      <td>Sr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1844</td>\n",
       "      <td>PS000002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>...</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Heard</td>\n",
       "      <td>Augustine</td>\n",
       "      <td>Sr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>ID0000001</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1844</td>\n",
       "      <td>PS000002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>...</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Heard</td>\n",
       "      <td>Augustine</td>\n",
       "      <td>Sr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>ID0000006</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Partner</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>1842</td>\n",
       "      <td>1847</td>\n",
       "      <td>PS000001;PS000002;PS000003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000003</td>\n",
       "      <td>...</td>\n",
       "      <td>23.130000</td>\n",
       "      <td>113.260000</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Dixwell</td>\n",
       "      <td>George B</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>239</td>\n",
       "      <td>ID0000126</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1873</td>\n",
       "      <td>1875</td>\n",
       "      <td>PS000017;PS000018;PS000019</td>\n",
       "      <td>Albert Heard's brother-in-law</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>...</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>H</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635</th>\n",
       "      <td>271</td>\n",
       "      <td>ID0000126</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>...</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Heard &amp; Co</td>\n",
       "      <td>USA</td>\n",
       "      <td>1876.0</td>\n",
       "      <td>1876</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>H</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>271</td>\n",
       "      <td>ID0000126</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1876</td>\n",
       "      <td>1876</td>\n",
       "      <td>PS000020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>...</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0020</td>\n",
       "      <td>Heard &amp; Co</td>\n",
       "      <td>USA</td>\n",
       "      <td>1876.0</td>\n",
       "      <td>1876</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>H</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>250</td>\n",
       "      <td>ID0000130</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1874</td>\n",
       "      <td>1874</td>\n",
       "      <td>PS000018</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>...</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Gillingham</td>\n",
       "      <td>A</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>254</td>\n",
       "      <td>ID0000132</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Clerk</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>1875</td>\n",
       "      <td>1875</td>\n",
       "      <td>PS000019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LC000017</td>\n",
       "      <td>...</td>\n",
       "      <td>34.690000</td>\n",
       "      <td>135.195556</td>\n",
       "      <td>CO0001</td>\n",
       "      <td>Augustine Heard &amp; Company</td>\n",
       "      <td>USA</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>1875</td>\n",
       "      <td>Hunt</td>\n",
       "      <td>J</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>639 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ID_x      Label Employer Occupation Location_x  ValidFrom  ValidTo  \\\n",
       "0       2  ID0000001   CO0001    Partner   LC000001       1845     1847   \n",
       "1       2  ID0000001   CO0001    Partner   LC000001       1845     1847   \n",
       "2      11  ID0000001   CO0001    Partner   LC000003       1842     1844   \n",
       "3      11  ID0000001   CO0001    Partner   LC000003       1842     1844   \n",
       "4       4  ID0000006   CO0001    Partner   LC000003       1842     1847   \n",
       "..    ...        ...      ...        ...        ...        ...      ...   \n",
       "634   239  ID0000126   CO0001      Clerk   LC000017       1873     1875   \n",
       "635   271  ID0000126   CO0020      Clerk   LC000017       1876     1876   \n",
       "636   271  ID0000126   CO0020      Clerk   LC000017       1876     1876   \n",
       "637   250  ID0000130   CO0001      Clerk   LC000017       1874     1874   \n",
       "638   254  ID0000132   CO0001      Clerk   LC000017       1875     1875   \n",
       "\n",
       "                         Source                          Notes    Loc_ID  ...  \\\n",
       "0             PS000001;PS000003                            NaN  LC000001  ...   \n",
       "1             PS000001;PS000003                            NaN  LC000001  ...   \n",
       "2                      PS000002                            NaN  LC000003  ...   \n",
       "3                      PS000002                            NaN  LC000003  ...   \n",
       "4    PS000001;PS000002;PS000003                            NaN  LC000003  ...   \n",
       "..                          ...                            ...       ...  ...   \n",
       "634  PS000017;PS000018;PS000019  Albert Heard's brother-in-law  LC000017  ...   \n",
       "635                    PS000020                            NaN  LC000017  ...   \n",
       "636                    PS000020                            NaN  LC000017  ...   \n",
       "637                    PS000018                            NaN  LC000017  ...   \n",
       "638                    PS000019                            NaN  LC000017  ...   \n",
       "\n",
       "           Lat         Lng    ID_y                CompanyName  \\\n",
       "0    42.360278  -71.057778  CO0001  Augustine Heard & Company   \n",
       "1    42.360278  -71.057778  CO0001  Augustine Heard & Company   \n",
       "2    23.130000  113.260000  CO0001  Augustine Heard & Company   \n",
       "3    23.130000  113.260000  CO0001  Augustine Heard & Company   \n",
       "4    23.130000  113.260000  CO0001  Augustine Heard & Company   \n",
       "..         ...         ...     ...                        ...   \n",
       "634  34.690000  135.195556  CO0001  Augustine Heard & Company   \n",
       "635  34.690000  135.195556  CO0020                 Heard & Co   \n",
       "636  34.690000  135.195556  CO0020                 Heard & Co   \n",
       "637  34.690000  135.195556  CO0001  Augustine Heard & Company   \n",
       "638  34.690000  135.195556  CO0001  Augustine Heard & Company   \n",
       "\n",
       "    NationalAffiliation DateFounded DateClosed     Surname  GivenName Suffix  \n",
       "0                   USA      1840.0       1875       Heard  Augustine     Sr  \n",
       "1                   USA      1840.0       1875       Heard  Augustine     Sr  \n",
       "2                   USA      1840.0       1875       Heard  Augustine     Sr  \n",
       "3                   USA      1840.0       1875       Heard  Augustine     Sr  \n",
       "4                   USA      1840.0       1875     Dixwell   George B         \n",
       "..                  ...         ...        ...         ...        ...    ...  \n",
       "634                 USA      1840.0       1875  Livingston          H         \n",
       "635                 USA      1876.0       1876  Livingston          H         \n",
       "636                 USA      1876.0       1876  Livingston          H         \n",
       "637                 USA      1840.0       1875  Gillingham          A         \n",
       "638                 USA      1840.0       1875        Hunt          J         \n",
       "\n",
       "[639 rows x 21 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_locations_companies = pd.merge(job_locations_companies, person[['Label','Surname', 'GivenName', 'Suffix']], how='inner', left_on='Label', right_on='Label')\n",
    "#job_locations_companies"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f3d1df7",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7364cb95",
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_dict(d):\n",
    "    \"\"\"\n",
    "    Strip all leading and trailing whitespace in dictionary keys and values.\n",
    "    \"\"\"\n",
    "    return dict((k.strip(), v.strip()) for k, v in d.items())\n",
    "\n",
    "savepath=\"json\"\n",
    "os.makedirs(savepath, exist_ok=True)\n",
    "\n",
    "for year in range(1842, 1892):\n",
    "    job_locations_companies_year = job_locations_companies[job_locations_companies.ValidFrom.le(\n",
    "        year) & job_locations_companies.ValidTo.ge(year)].copy()\n",
    "    companies_subset = companies[companies['ID'].isin(\n",
    "        set(job_locations_companies_year.ID_y))]\n",
    "    locations_subset = locations[locations.Loc_ID.isin(\n",
    "        set(job_locations_companies_year.Loc_ID))]\n",
    "    ##\n",
    "    person_subset_builder1 = person[person.Label.isin(\n",
    "        set(job_locations_companies_year.Label))]\n",
    "    person_subset = pd.merge(person_subset_builder1, job_locations_companies_year[[\n",
    "                             'Label', 'Occupation', 'Location_y', 'CompanyName']], how='inner', left_on='Label', right_on='Label')\n",
    "    # person_subset['ID'] = (person_subset['GivenName'].values + ' ' + person_subset['Surname'].values + ' ' + person_subset['Suffix'].fillna('').astype(str).values);\n",
    "    # person_subset.drop(columns=['Location', 'Employer'], inplace=True)\n",
    "    # person_subset.rename(columns={\"Location_y\": \"Location\", \"CompanyName\": \"Company\"}, inplace=True)\n",
    "    ##\n",
    "    edges_subset = edges[(edges.Source.isin(set(job_locations_companies_year.ID_x))) & (\n",
    "        edges.Target.isin(set(job_locations_companies_year.ID_x)))]\n",
    "\n",
    "    # Create graph\n",
    "    G = nx.Graph()\n",
    "    for n, row in companies_subset.iterrows():\n",
    "        details = row.to_dict()\n",
    "        keys = ['ID']\n",
    "        details = {key: details[key] for key in details if key not in keys}\n",
    "        details['Company'] = details.pop('CompanyName')\n",
    "        # G.add_node(row['ID'], node_type=\"company\", details=details)\n",
    "    for n, row in locations_subset.iterrows():\n",
    "        details = row.to_dict()\n",
    "        keys = ['Loc_ID']\n",
    "        details = {key: details[key] for key in details if key not in keys}\n",
    "        # G.add_node(row['Location'], node_type=\"location\", details=details)\n",
    "    for n, row in person_subset.iterrows():\n",
    "        details = row.to_dict()\n",
    "        keys = ['Label', 'Employer', 'Location']\n",
    "        details = {key: details[key] for key in details if key not in keys}\n",
    "        details['Location'] = details.pop('Location_y')\n",
    "        details['Company'] = details.pop('CompanyName')\n",
    "        G.add_node(row['Label'], details=details)\n",
    "    for n, row in edges_subset.iterrows():\n",
    "        source_name = job_locations_companies_year[[\"GivenName\", \"Surname\", \"Suffix\"]][job_locations_companies_year['ID_x']\n",
    "                                                       == row.Source].iloc[0]\n",
    "        sourceName = (source_name['GivenName'] + ' ' + source_name['Surname'] + ' ' + source_name['Suffix']).strip()\n",
    "        source = job_locations_companies_year['Label'][job_locations_companies_year['ID_x']\n",
    "                                                       == row.Source].iloc[0]\n",
    "        target = job_locations_companies_year['Label'][job_locations_companies_year['ID_x']\n",
    "                                                       == row.Target].iloc[0]\n",
    "        target_name = job_locations_companies_year[[\"GivenName\", \"Surname\", \"Suffix\"]][job_locations_companies_year['ID_x']\n",
    "                                                       == row.Target].iloc[0]\n",
    "        targetName = (target_name['GivenName'] + ' ' + target_name['Surname'] + ' ' + target_name['Suffix']).strip()\n",
    "        \n",
    "        G.add_edge(source, target, details={\n",
    "                   'weight', 1}, source_id=sourceName, target_id=targetName)                   \n",
    "    # Companies\n",
    "    #for n, row in job_locations_companies_year.iterrows():\n",
    "    #    emp = row.get('Employer')\n",
    "    #    if(emp and emp is not np.nan):\n",
    "    #        pid = row['Label']\n",
    "    #        m = job_locations_companies_year[(job_locations_companies_year['Label'] == pid) & (\n",
    "    #            job_locations_companies_year['Employer'] == emp)]\n",
    "    #        G.add_edge(pid, emp, weight=int(m.size * 5),\n",
    "    #                   source_type='person', target_type='company')\n",
    "    # Locations\n",
    "    #for n, row in job_locations_companies_year.iterrows():\n",
    "    #    locationy = row.get('Location_y')\n",
    "    #    if(locationy and locationy is not np.nan):\n",
    "    #        pid = row['Label']\n",
    "    #        m = job_locations_companies_year[(job_locations_companies_year['Label'] == pid) & (\n",
    "    #            job_locations_companies_year['Location_y'] == locationy)]\n",
    "    #        G.add_edge(pid, locationy, weight=int(m.size * 5),\n",
    "    #                   source_type='person', target_type='location')\n",
    "    # Company to Locations\n",
    "    #for n, row in job_locations_companies_year.iterrows():\n",
    "    #    locationy = row.get('Location_y')\n",
    "    #    emp = row.get('Employer')\n",
    "    #    if(emp and emp is not np.nan and locationy and locationy is not np.nan):\n",
    "    #        m = job_locations_companies_year[(job_locations_companies_year['Employer'] == emp) & (\n",
    "    #            job_locations_companies_year['Location_y'] == locationy)]\n",
    "    #        print(emp)\n",
    "    #        print(locationy)\n",
    "    #        G.add_edge(emp, locationy, weight=int(m.size * 5),\n",
    "    #                   source_type='company', target_type='location')\n",
    "    pos = {}\n",
    "    # for node in G.nodes(data=True):\n",
    "    #   nodename = node[0]\n",
    "    #   print(nodename)\n",
    "    #   if node[1]['node_type'] == 'company':\n",
    "    #     print(node[1])\n",
    "    #   if node[1]['node_type'] == 'location':\n",
    "    #     print(node[1])\n",
    "    #   if node[1]['node_type'] == 'person':\n",
    "    #     print(node[1])\n",
    "\n",
    "    NodeLabels = pd.Series((person_subset['GivenName'].values + ' ' + person_subset['Surname'].values +\n",
    "                           ' ' + person_subset['Suffix'].fillna('').astype(str).values), index=person_subset.Label).to_dict()\n",
    "    NodeLabels = strip_dict(NodeLabels)\n",
    "    # Add Companies\n",
    "    NodeLabels.update(pd.Series(\n",
    "        companies_subset['CompanyName'].values, index=companies_subset.ID).to_dict())\n",
    "    # Add Locations\n",
    "    NodeLabels.update(pd.Series(\n",
    "        locations_subset['Location'].values, index=locations_subset.Location).to_dict())\n",
    "    G = nx.relabel_nodes(G, NodeLabels)\n",
    "    d = nx.json_graph.node_link_data(G)\n",
    "    converted = json.dumps(d, cls=SetEncoder, indent=4)\n",
    "    with open(os.path.join(savepath, str(year) + '.json'), 'w') as f:\n",
    "        f.write(converted)\n",
    "    G.clear()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "597c51c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "distinctCompanies = job_locations_companies['CompanyName'].dropna(\n",
    ").unique().tolist()\n",
    "distinctOccupations = job_locations_companies['Occupation'].dropna(\n",
    ").unique().tolist()\n",
    "distinctLocations = job_locations_companies['Location_y'].dropna(\n",
    ").unique().tolist()\n",
    "distinctNationalities = person['Nationality'].dropna(\n",
    ").unique().tolist()\n",
    "distinctNationalAffiliations = job_locations_companies['NationalAffiliation'].dropna(\n",
    ").unique().tolist()\n",
    "\n",
    "distinctCompanies.sort()\n",
    "distinctOccupations.sort()\n",
    "distinctLocations.sort()\n",
    "distinctNationalities.sort()\n",
    "distinctNationalAffiliations.sort()\n",
    "\n",
    "selections = {\n",
    "    \"companies\": distinctCompanies,\n",
    "    \"nationalaffiliations\": distinctNationalAffiliations,\n",
    "    \"locations\": distinctLocations,\n",
    "    \"occupations\": distinctOccupations,\n",
    "    \"nationalities\": distinctNationalities\n",
    "}\n",
    "#json.dumps(selections)\n",
    "\n",
    "\n",
    "\n",
    "with open(os.path.join(savepath, \"selections.json\"), \"w\") as f:\n",
    "    json.dump(selections, f, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9b329e6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"Boston\":{\n",
      "        \"Lat\":42.360278,\n",
      "        \"Lng\":-71.057778\n",
      "    },\n",
      "    \"Hong Kong\":{\n",
      "        \"Lat\":22.3,\n",
      "        \"Lng\":114.2\n",
      "    },\n",
      "    \"Guangzhou\":{\n",
      "        \"Lat\":23.13,\n",
      "        \"Lng\":113.26\n",
      "    },\n",
      "    \"Shanghai\":{\n",
      "        \"Lat\":31.228611,\n",
      "        \"Lng\":121.474722\n",
      "    },\n",
      "    \"Macau\":{\n",
      "        \"Lat\":22.166667,\n",
      "        \"Lng\":113.55\n",
      "    },\n",
      "    \"London\":{\n",
      "        \"Lat\":51.507222,\n",
      "        \"Lng\":-0.1275\n",
      "    },\n",
      "    \"Cairo\":{\n",
      "        \"Lat\":30.044444,\n",
      "        \"Lng\":31.235833\n",
      "    },\n",
      "    \"Monaco\":{\n",
      "        \"Lat\":43.731111,\n",
      "        \"Lng\":7.42\n",
      "    },\n",
      "    \"Ningbo\":{\n",
      "        \"Lat\":29.8603,\n",
      "        \"Lng\":121.6245\n",
      "    },\n",
      "    \"Fuzhou\":{\n",
      "        \"Lat\":26.0743,\n",
      "        \"Lng\":119.2964\n",
      "    },\n",
      "    \"Xiamen\":{\n",
      "        \"Lat\":24.4796,\n",
      "        \"Lng\":118.0889\n",
      "    },\n",
      "    \"Hankou\":{\n",
      "        \"Lat\":30.581179,\n",
      "        \"Lng\":114.272597\n",
      "    },\n",
      "    \"Jiujiang\":{\n",
      "        \"Lat\":29.661,\n",
      "        \"Lng\":115.954\n",
      "    },\n",
      "    \"Tianjin\":{\n",
      "        \"Lat\":39.1336,\n",
      "        \"Lng\":117.2054\n",
      "    },\n",
      "    \"Yokohama\":{\n",
      "        \"Lat\":35.444167,\n",
      "        \"Lng\":139.638056\n",
      "    },\n",
      "    \"Bangkok\":{\n",
      "        \"Lat\":13.7525,\n",
      "        \"Lng\":100.494167\n",
      "    },\n",
      "    \"Kobe\":{\n",
      "        \"Lat\":34.69,\n",
      "        \"Lng\":135.195556\n",
      "    },\n",
      "    \"Beijing\":{\n",
      "        \"Lat\":39.906667,\n",
      "        \"Lng\":116.3975\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "location_json = locations[['Location', 'Lat', 'Lng']].copy().set_index('Location').to_json(orient=\"index\", indent=4)\n",
    "#print(location_json)\n",
    "with open(os.path.join(savepath, \"locations.json\"), \"w\") as f:\n",
    "    f.write(location_json)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de8828fa",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit ('3.9.13')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "13d993c62481d811c91b5e2a808f525c1a9fcbcd5c590426656215ae8cc27797"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}