prepare_people.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import html\n",
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "import unidecode\n",
    "\n",
    "from os.path import join\n",
    "\n",
    "DATA_DIR = 'raw_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"pid\",\"firstname\",\"middlename\",\"lastname\",\"degrees\",\"location\",\"locid\",\"area\",\"majorarea\",\"award\",\"hindex\",\"orcid_id\",\"s2id\",\"homepage\",\"addedby\",\"dateadded\",\"modby\"\r\n",
      "\"1\",\"Stephen\",\"V.\",\"David\",\"Ph.D.\",\"Oregon Health and Science University\",\"226\",\"auditory system, attention\",\"neuro,csd,bme\",\"\",\"30\",\"0000-0003-4135-3104\",\"2321659\",\"http://hearingbrain.org/\",\"david\",\"2005-01-14 15:50:37\",\"david\"\r\n",
      "\"3\",\"Jack\",\"L.\",\"Gallant\",\"\",\"University of California, Berkeley\",\"312\",\"Systems\",\"neuro,psych\",\"\",\"46\",\"NULL\",\"40373111\",\"http://gallantlab.org\",\"david\",\"2005-01-14 15:51:51\",\"david\"\r\n",
      "\"667145\",\"Christian\",\"T.\",\"Totten\",\"Ph.D.\",\"University of Florida, Gainesville\",\"380\",\"Environmental Engineering\",\"etree,environment\",\"NULL\",\"2\",\"NULL\",\"35259558\",\"\",\"pq\",\"2016-07-23 11:14:13\",\"pq\"\r\n",
      "\"5\",\"Kendrick\",\"Norris\",\"Kay\",\"Ph.D.\",\"University of Minnesota, Twin Cities\",\"406\",\"visual system, fMRI, computational methods, neural network models\",\"neuro\",\"\",\"24\",\"\",\"1912661\",\"http://cvnlab.net\",\"david\",\"2005-01-15 18:20:41\",\"kendrick\"\r\n",
      "\"6\",\"Ryan\",\"\",\"Prenger\",\"\",\"Baidu\",\"33232\",\"visual system\",\"neuro\",\"\",\"7\",\"\",\"3283879\",\"\",\"david\",\"2005-01-15 18:23:32\",\"david\"\r\n"
     ]
    }
   ],
   "source": [
    "! head -6 raw_data/people.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(778367, 9)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def load_author_df():\n",
    "    \"\"\"\"\"\"\n",
    "    result = []\n",
    "    with open(join(DATA_DIR, 'people.csv')) as fin:\n",
    "        reader = csv.reader(fin, delimiter=',')\n",
    "        next(reader, None)\n",
    "        for row in reader:\n",
    "            result.append(list(row[:7]) + [row[8]] + [row[11]])\n",
    "    col = ['pid', 'firstname', 'middlename', 'lastname', 'degrees', 'location', 'locid', 'majorarea', 'orcid']\n",
    "    df = pd.DataFrame(sorted(result, key=lambda x: int(x[0])), columns=col)\n",
    "    for c in col[1:]:\n",
    "        df[c] = df[c].apply(lambda x: '' if x == 'NULL' else x)\n",
    "    return df\n",
    "\n",
    "people_df = load_author_df()\n",
    "people_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>firstname</th>\n",
       "      <th>middlename</th>\n",
       "      <th>lastname</th>\n",
       "      <th>degrees</th>\n",
       "      <th>location</th>\n",
       "      <th>locid</th>\n",
       "      <th>majorarea</th>\n",
       "      <th>orcid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>V.</td>\n",
       "      <td>David</td>\n",
       "      <td>Ph.D.</td>\n",
       "      <td>Oregon Health and Science University</td>\n",
       "      <td>226</td>\n",
       "      <td>neuro,csd,bme</td>\n",
       "      <td>0000-0003-4135-3104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Benjamin</td>\n",
       "      <td>Y.</td>\n",
       "      <td>Hayden</td>\n",
       "      <td></td>\n",
       "      <td>University of Minnesota, Twin Cities</td>\n",
       "      <td>406</td>\n",
       "      <td>neuro</td>\n",
       "      <td>0000-0002-7678-4281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Jack</td>\n",
       "      <td>L.</td>\n",
       "      <td>Gallant</td>\n",
       "      <td></td>\n",
       "      <td>University of California, Berkeley</td>\n",
       "      <td>312</td>\n",
       "      <td>neuro,psych</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Benjamin</td>\n",
       "      <td></td>\n",
       "      <td>Willmore</td>\n",
       "      <td>Ph.D.</td>\n",
       "      <td>University of Oxford</td>\n",
       "      <td>228</td>\n",
       "      <td>neuro</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Kendrick</td>\n",
       "      <td>Norris</td>\n",
       "      <td>Kay</td>\n",
       "      <td>Ph.D.</td>\n",
       "      <td>University of Minnesota, Twin Cities</td>\n",
       "      <td>406</td>\n",
       "      <td>neuro</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pid firstname middlename  lastname degrees  \\\n",
       "0   1   Stephen         V.     David   Ph.D.   \n",
       "1   2  Benjamin         Y.    Hayden           \n",
       "2   3      Jack         L.   Gallant           \n",
       "3   4  Benjamin             Willmore   Ph.D.   \n",
       "4   5  Kendrick     Norris       Kay   Ph.D.   \n",
       "\n",
       "                               location locid      majorarea  \\\n",
       "0  Oregon Health and Science University   226  neuro,csd,bme   \n",
       "1  University of Minnesota, Twin Cities   406          neuro   \n",
       "2    University of California, Berkeley   312    neuro,psych   \n",
       "3                  University of Oxford   228          neuro   \n",
       "4  University of Minnesota, Twin Cities   406          neuro   \n",
       "\n",
       "                 orcid  \n",
       "0  0000-0003-4135-3104  \n",
       "1  0000-0002-7678-4281  \n",
       "2                       \n",
       "3                       \n",
       "4                       "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "778367"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_df.pid.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "603621    1\n",
       "825408    1\n",
       "343844    1\n",
       "58824     1\n",
       "4609      1\n",
       "         ..\n",
       "200159    1\n",
       "106807    1\n",
       "174614    1\n",
       "414197    1\n",
       "530467    1\n",
       "Name: pid, Length: 778367, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_df.pid.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('pid', 0),\n",
       " ('firstname', 0),\n",
       " ('middlename', 0),\n",
       " ('lastname', 0),\n",
       " ('degrees', 0),\n",
       " ('location', 0),\n",
       " ('locid', 0),\n",
       " ('majorarea', 0),\n",
       " ('orcid', 0)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(c, people_df[c].isna().sum()) for c in people_df.columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter PIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('pid', 0),\n",
       " ('firstname', 567),\n",
       " ('middlename', 364406),\n",
       " ('lastname', 1110),\n",
       " ('degrees', 319825),\n",
       " ('location', 11482),\n",
       " ('locid', 13),\n",
       " ('majorarea', 0),\n",
       " ('orcid', 776686)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(c, (people_df[c] == '').sum()) for c in people_df.columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without first or last name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "567"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pid_wo_first = set(people_df[people_df.firstname == '']['pid'])\n",
    "len(pid_wo_first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1110"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pid_wo_last = set(people_df[people_df.lastname == '']['pid'])\n",
    "len(pid_wo_last)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(776975, 9)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = people_df[~people_df.pid.isin(pid_wo_first | pid_wo_last)].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Same name and affiliation with multiple PIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "776110"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inst_to_pids = ds_people.groupby(\n",
    "        ['firstname', 'middlename', 'lastname', 'location', 'majorarea'])['pid'].apply(set).to_dict()\n",
    "len(inst_to_pids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "824"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inst_with_mul_pids = [k for k, v in inst_to_pids.items() if len(v) > 1]\n",
    "len(inst_with_mul_pids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('A',\n",
       " 'Peter',\n",
       " 'Klimley',\n",
       " 'University of California, Davis and San Diego State University',\n",
       " 'physics')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inst_with_mul_pids[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>firstname</th>\n",
       "      <th>middlename</th>\n",
       "      <th>lastname</th>\n",
       "      <th>degrees</th>\n",
       "      <th>location</th>\n",
       "      <th>locid</th>\n",
       "      <th>majorarea</th>\n",
       "      <th>orcid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>595864</th>\n",
       "      <td>635445</td>\n",
       "      <td>A</td>\n",
       "      <td>Peter</td>\n",
       "      <td>Klimley</td>\n",
       "      <td></td>\n",
       "      <td>University of California, Davis and San Diego ...</td>\n",
       "      <td>22646</td>\n",
       "      <td>physics</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595865</th>\n",
       "      <td>635446</td>\n",
       "      <td>A</td>\n",
       "      <td>Peter</td>\n",
       "      <td>Klimley</td>\n",
       "      <td></td>\n",
       "      <td>University of California, Davis and San Diego ...</td>\n",
       "      <td>22646</td>\n",
       "      <td>physics</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pid firstname middlename lastname degrees  \\\n",
       "595864  635445         A      Peter  Klimley           \n",
       "595865  635446         A      Peter  Klimley           \n",
       "\n",
       "                                                 location  locid majorarea  \\\n",
       "595864  University of California, Davis and San Diego ...  22646   physics   \n",
       "595865  University of California, Davis and San Diego ...  22646   physics   \n",
       "\n",
       "       orcid  \n",
       "595864        \n",
       "595865        "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people[\n",
    "    (ds_people.firstname == 'A') & (ds_people.middlename == 'Peter') & (ds_people.lastname == 'Klimley')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1689"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mul_pids = frozenset().union(*[inst_to_pids[k] for k in inst_with_mul_pids])\n",
    "len(mul_pids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775286, 9)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = ds_people[~ds_people.pid.isin(mul_pids)].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Last name with one character"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775118, 9)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = ds_people[ds_people.lastname.str.len() > 1].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JUYUN\n",
      "Max\n",
      "BENJAMIN\n",
      "JAMES\n",
      "CABRERA\n",
      "NICHOLAS\n",
      "MANSI\n",
      "JOHN\n",
      "YUAN \n",
      "JAMES\n",
      "FRANZ LUDWIG\n",
      "EDWARD\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def norm_firstname(x):\n",
    "    \"\"\"\"\"\"\n",
    "    x = re.sub(r'[\\000-\\010]|[\\013-\\014]|[\\016-\\037]', '', x).replace('\\\\t', '').strip()\n",
    "    if x[0] == '(' and x[-1] == ')':\n",
    "        return x[1:-1]\n",
    "    x = x.rstrip('|').strip()\n",
    "    x = re.sub(r'\\([^)]*\\)', '', x).lstrip('(')\n",
    "    if ',' in x:\n",
    "        x = x.split(',')[0]\n",
    "    x = x.replace('<q>', '\"').replace('</q>', '\"')\n",
    "    x = x.replace('<i>', '').replace('</i>', '')\n",
    "    x = re.sub(r'\"[^)]*\"', '', x)\n",
    "    x = html.unescape(x).replace('.', ' ')\n",
    "    x = ' '.join(x.split()).strip()\n",
    "    x = unidecode.unidecode(x).upper()\n",
    "    if x != 'SIR' and x.startswith('SIR '):\n",
    "        return x[4:]\n",
    "    return x\n",
    "\n",
    "print(\n",
    "    norm_firstname('Juyun \\\\t'),\n",
    "    norm_firstname('(Max)'),\n",
    "    norm_firstname('Benjamin'),\n",
    "    norm_firstname('James, Jr.'),\n",
    "    norm_firstname('Cabrera,'),\n",
    "    norm_firstname('(Nicholas'),\n",
    "    norm_firstname('Mansi \"Indian Princess\"'),\n",
    "    norm_firstname('John (\"Jack\")'),\n",
    "    norm_firstname('&#34945;'),\n",
    "    norm_firstname('James <q>Brad</q>'),\n",
    "    norm_firstname('Franz <i>Ludwig</i>'),\n",
    "    norm_firstname('\\x03\\x14\\x15\\x13\\x0b\\x14\\x04Edward'),\n",
    "    norm_firstname(','),\n",
    "    sep='\\n'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775118, 10)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people['firstname_norm'] = ds_people.firstname.apply(norm_firstname)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' ',\n",
       " \"'\",\n",
       " '(',\n",
       " ')',\n",
       " '-',\n",
       " '/',\n",
       " '0',\n",
       " '1',\n",
       " '2',\n",
       " '5',\n",
       " '8',\n",
       " ':',\n",
       " '>',\n",
       " '?',\n",
       " '[',\n",
       " ']',\n",
       " '|'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(x for e in ds_people.firstname_norm for x in e.lower() if x not in set(string.ascii_lowercase))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5358 Mani/Madhuri MANI/MADHURI\n",
      "5360 Jenny/Yun-Chia JENNY/YUN-CHIA\n",
      "19835 Hyun Ju [Helena] HYUN JU [HELENA]\n",
      "58000 ¹Ü 1U\n",
      "58150 »Æ >>AE\n",
      "58151 Â½ A 1/2 \n",
      "58152 »ª >>A\n",
      "90933 Pumpki [Lei] PUMPKI [LEI]\n",
      "196545 Catherine|Garth CATHERINE|GARTH\n",
      "220009 Richard|Rinpoche RICHARD|RINPOCHE\n",
      "229826 Laurel|Murakawa-Leopard LAUREL|MURAKAWA-LEOPARD\n",
      "252888 Jonathan|Buell JONATHAN|BUELL\n",
      "252889 Janelle|Merchant JANELLE|MERCHANT\n",
      "252896 Kristie|Salinas KRISTIE|SALINAS\n",
      "252897 Andrew|Centner ANDREW|CENTNER\n",
      "252898 Matthew|Lamar MATTHEW|LAMAR\n",
      "252899 Jeff|Cerone JEFF|CERONE\n",
      "255271 Anne|Brack ANNE|BRACK\n",
      "255281 Jennifer|Clark JENNIFER|CLARK\n",
      "262891 Lindsay|Knickman LINDSAY|KNICKMAN\n",
      "262892 Gabrielle|Knickman GABRIELLE|KNICKMAN\n",
      "262893 Kevin|Schulte KEVIN|SCHULTE\n",
      "282180 Michele|Counts MICHELE|COUNTS\n",
      "282192 Georgia|Kimball GEORGIA|KIMBALL\n",
      "282199 Sarah|Pettengill SARAH|PETTENGILL\n",
      "282200 Ashley|Meyers ASHLEY|MEYERS\n",
      "282202 Michelle|Harms MICHELLE|HARMS\n",
      "282203 Steven|Schmitz STEVEN|SCHMITZ\n",
      "282204 Rebekah|Kiene REBEKAH|KIENE\n",
      "282205 Brian|Denney BRIAN|DENNEY\n",
      "282207 David|Beagle DAVID|BEAGLE\n",
      "282208 Alan|Kiene ALAN|KIENE\n",
      "282209 Jeremy|Siebert JEREMY|SIEBERT\n",
      "282210 Virgil|Day VIRGIL|DAY\n",
      "282212 Pat|Day PAT|DAY\n",
      "282213 Todd|Plowman TODD|PLOWMAN\n",
      "282214 Corey|Siebert COREY|SIEBERT\n",
      "282215 Bonnie|Miller BONNIE|MILLER\n",
      "282227 Matt|Haley MATT|HALEY\n",
      "282232 Rhonda|Kirkpatrick RHONDA|KIRKPATRICK\n",
      "282233 Susan|Hardee SUSAN|HARDEE\n",
      "282234 Judy|Manning JUDY|MANNING\n",
      "282235 Gary|Hedrick GARY|HEDRICK\n",
      "282236 Missy|Manning MISSY|MANNING\n",
      "295261 Erica|Casavant ERICA|CASAVANT\n",
      "309433 Michelle|Stanley MICHELLE|STANLEY\n",
      "317679 Ross|Beaudoin ROSS|BEAUDOIN\n",
      "321436 Allison|Krause ALLISON|KRAUSE\n",
      "321438 Dorothy|Hayden DOROTHY|HAYDEN\n",
      "321440 Jennifer|Deist JENNIFER|DEIST\n",
      "321441 Carol|Siebe CAROL|SIEBE\n",
      "321446 Jim|Sparks JIM|SPARKS\n",
      "321449 Erin|Griffith ERIN|GRIFFITH\n",
      "321451 John|Bloom JOHN|BLOOM\n",
      "321452 Danny|Bloom DANNY|BLOOM\n",
      "322157 Becky|Carter BECKY|CARTER\n",
      "324860 Marc|Murray MARC|MURRAY\n",
      "326657 Thomas|Flynn THOMAS|FLYNN\n",
      "326659 James|Floyd JAMES|FLOYD\n",
      "326660 James|James JAMES|JAMES\n",
      "326661 Lisa|Perkins LISA|PERKINS\n",
      "326662 Dorothy|Poe DOROTHY|POE\n",
      "326663 Randolph|Young RANDOLPH|YOUNG\n",
      "341759 Nathan|Buchanan NATHAN|BUCHANAN\n",
      "341760 Tenielle|Lang TENIELLE|LANG\n",
      "341763 Andrew|Frogge ANDREW|FROGGE\n",
      "341764 Laura-Lee|Lang LAURA-LEE|LANG\n",
      "341766 Heather|Henderson HEATHER|HENDERSON\n",
      "341767 Traci|Jones TRACI|JONES\n",
      "341768 Melinda|Philbin MELINDA|PHILBIN\n",
      "341769 Tony|Holdren TONY|HOLDREN\n",
      "341770 Danielle|Majors DANIELLE|MAJORS\n",
      "341771 Janet|Jones JANET|JONES\n",
      "341859 Shelley|Bacigalupo SHELLEY|BACIGALUPO\n",
      "341860 Karen|Karraker KAREN|KARRAKER\n",
      "341861 Kayleigh|Bacigalupo KAYLEIGH|BACIGALUPO\n",
      "342850 Adrian|Seiler ADRIAN|SEILER\n",
      "343561 Tesia|Duplessis TESIA|DUPLESSIS\n",
      "401153 .. \n",
      "510996 Elda|Mulhall ELDA|MULHALL\n",
      "603930 Zach|Webster ZACH|WEBSTER\n",
      "604466 Dean|Crawford DEAN|CRAWFORD\n",
      "604467 Joshua|Baldwin JOSHUA|BALDWIN\n",
      "604468 Jacob|Baldwin JACOB|BALDWIN\n",
      "604469 Jonathan|Crawford JONATHAN|CRAWFORD\n",
      "604470 Jonathan|Baldwin JONATHAN|BALDWIN\n",
      "604471 Semin|Baldwin SEMIN|BALDWIN\n",
      "604472 Justine|Baldwin JUSTINE|BALDWIN\n",
      "612477 Eric|Barney ERIC|BARNEY\n",
      "612478 Jared|Barney JARED|BARNEY\n",
      "612479 Justin|Barney JUSTIN|BARNEY\n",
      "641672 Avdviser: AVDVISER:\n",
      "652235 Zaâ€™ndra ZAAEUR(TM)NDRA\n",
      "660755 Advisr: ADVISR:\n",
      "700702 Fai͏̈za FAI[?]ZA\n",
      "704190 Lhoussai͏̈ne LHOUSSAI[?]NE\n",
      "706157 Essai͏̈d ESSAI[?]D\n",
      "811119 . \n",
      "818444 , \n",
      "823011 5280 5280\n",
      "824156 1800 Get a Rug 1800 GET A RUG\n"
     ]
    }
   ],
   "source": [
    "def print_dirty_firstname():\n",
    "    \"\"\"\"\"\"\n",
    "    result = []\n",
    "    char = set(['(', ')', '/', '0', '1', '2', '5', '8', ':', '>', '?', '[', '\\\\', ']', '|'])\n",
    "    for pid, f, fn in zip(ds_people.pid, ds_people.firstname, ds_people.firstname_norm):\n",
    "        if fn == '' or len(set(fn) & char) > 0:\n",
    "            print(pid, f, fn)\n",
    "            result.append(pid)\n",
    "    return result\n",
    "\n",
    "dirty_fn_pid = print_dirty_firstname()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775017, 10)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = ds_people[~ds_people.pid.isin(dirty_fn_pid)].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Middle name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MACFARLAND\n",
      "RENE\n",
      "SHONA\n",
      "LI\n",
      "\n",
      "FONSECA\n",
      "M A\n",
      "L U\n",
      "JOSE\n",
      "L\n",
      "L\n",
      "R\n",
      "KAMIL UGURBIL\n",
      "LIN JIE XUAN\n",
      "T J\n",
      "THOMAS P-N\n",
      "MATEJ\n"
     ]
    }
   ],
   "source": [
    "def norm_middlename(x):\n",
    "    \"\"\"\"\"\"\n",
    "    if x in ['', '(', '1st', '2nd', '/', '$', '`', \"'\", '&', '(.']:\n",
    "        return ''\n",
    "    if x[0] == '(' and x[-1] == ')':\n",
    "        x = x[1:-1]\n",
    "    elif x[0] == \"'\" and x[-1] == \"'\":\n",
    "        x = x[1:-1]\n",
    "    elif x[0] == '[' and x[-1] == ']':\n",
    "        x = x[1:-1]\n",
    "    if ',' in x:\n",
    "        x = x.split(',')[0]\n",
    "    x = x.replace('_', ' ')\n",
    "    x = x.replace('<q>', '').replace('</q>', '')\n",
    "    x = x.replace('<i>', '').replace('</i>', '')\n",
    "    x = unidecode.unidecode(html.unescape(x)).upper()\n",
    "    x = re.sub(r'\\([^)]*\\)', '', x)\n",
    "    x = re.sub(r'\\\"[^)]*\\\"', '', x)\n",
    "    x = x.replace('.-', '-').replace('.', ' ')\n",
    "    x = ' '.join(x.split()).lstrip('(').rstrip(\"'\").strip()\n",
    "    return x\n",
    "\n",
    "print(\n",
    "    norm_middlename('[MacFarland]'),\n",
    "    norm_middlename(\"Rene'\"),\n",
    "    norm_middlename('<q>Shona</q>'),\n",
    "    norm_middlename('Li,'),\n",
    "    norm_middlename('('),\n",
    "    norm_middlename('Fonseca'),\n",
    "    norm_middlename('M. A.'),\n",
    "    norm_middlename('L.U.'),\n",
    "    norm_middlename('José'),\n",
    "    norm_middlename('L. (Molly)'),\n",
    "    norm_middlename('L. \"Lee\"'),\n",
    "    norm_middlename('R. (Rip)'),\n",
    "    norm_middlename('(Kâmil Uğurbil)'),\n",
    "    norm_middlename('(林節玄)'),\n",
    "    norm_middlename('(T.J.)'),\n",
    "    norm_middlename('Thomas P.-N'),\n",
    "    norm_middlename('Mat&#283;j'),\n",
    "    sep='\\n'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775017, 11)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people['middlename_norm'] = ds_people.middlename.apply(norm_middlename)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' ', '&', \"'\", '-', '/', '1', '3', '4', '7', '9', '<', '?', '\\\\'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(x for e in ds_people.middlename_norm for x in e.lower() if x not in set(string.ascii_lowercase))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57998 ---> ¹ú ---> 1U\n",
      "57999 ---> Ì« ---> I<<\n",
      "58149 ---> Ë¾ ---> E 3/4\n",
      "74675 ---> X/ ---> X/\n",
      "76348 ---> ? ---> ?\n",
      "80184 ---> O\\\\'neal ---> O\\\\'NEAL\n",
      "664979 ---> s/o ---> S/O\n",
      "693244 ---> (1973-....) ---> 1973-\n",
      "725710 ---> F/Ferreira ---> F/FERREIRA\n",
      "761071 ---> James/J. ---> JAMES/J\n",
      "825930 ---> A&M ---> A&M\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def print_dirty_middlename():\n",
    "    \"\"\"\"\"\"\n",
    "    result = []\n",
    "    char = set(\n",
    "        ['$', '&', '(', ',', '/', '1', '2', '3', '4', '7', '9', '<', '>', '?', '[', '\\\\', ']', '_', '`'])\n",
    "    for pid, m, mn in zip(ds_people.pid, ds_people.middlename, ds_people.middlename_norm):\n",
    "        if len(set(mn) & char) > 0:\n",
    "            print(pid, m, mn, sep=' ---> ')\n",
    "            result.append(pid)\n",
    "    return result\n",
    "\n",
    "dirty_mn_pid = print_dirty_middlename()\n",
    "len(dirty_mn_pid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775006, 11)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = ds_people[~ds_people.pid.isin(dirty_mn_pid)].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Last name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KUJALA\n",
      "WANG\n"
     ]
    }
   ],
   "source": [
    "def norm_lastname(x):\n",
    "    \"\"\"\"\"\"\n",
    "    if x in ['Ph.D.', 'Jr.', 'M.D.']:\n",
    "        return ''\n",
    "    if x[0] == '(' and x[-1] == ')':\n",
    "        x = x[1:-1]\n",
    "    if ',' in x:\n",
    "        x = x.split(',')[0]\n",
    "    x = html.unescape(x).upper()\n",
    "    x = re.sub(r'\\([^)]*\\)', '', x).lstrip('(')\n",
    "    x = ' '.join(x.split()).strip()\n",
    "    x = unidecode.unidecode(x).strip('.').replace('.', '')\n",
    "    return x\n",
    "\n",
    "print(\n",
    "    norm_lastname('Kujala (neé Saarela)'),\n",
    "    norm_lastname('Wang (&#29579;)'),\n",
    "    sep='\\n'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775006, 12)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people['lastname_norm'] = ds_people.lastname.apply(norm_lastname)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'\\x1a',\n",
       " ' ',\n",
       " '\"',\n",
       " '&',\n",
       " \"'\",\n",
       " '(',\n",
       " ')',\n",
       " '+',\n",
       " '-',\n",
       " '/',\n",
       " '0',\n",
       " '1',\n",
       " '2',\n",
       " '3',\n",
       " '4',\n",
       " '5',\n",
       " '6',\n",
       " ':',\n",
       " ';',\n",
       " '<',\n",
       " '>',\n",
       " '?',\n",
       " '[',\n",
       " '\\\\',\n",
       " ']',\n",
       " '_',\n",
       " '`'}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(x for e in ds_people.lastname_norm for x in e.lower() if x not in set(string.ascii_lowercase))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7067 ---> [Holley] Miner ---> [HOLLEY] MINER\n",
      "7071 ---> [Himmelheber] Brady ---> [HIMMELHEBER] BRADY\n",
      "18265 ---> O\\\\'Brien ---> O\\\\'BRIEN\n",
      "23940 ---> Collins 3rd ---> COLLINS 3RD\n",
      "28922 ---> O\"Riordan ---> O\"RIORDAN\n",
      "37799 ---> IÃ±iguez ---> IA+-IGUEZ\n",
      "55980 ---> La_Marca ---> LA_MARCA\n",
      "58153 ---> »ª ---> >>a\n",
      "58154 ---> ´« ---> '<<\n",
      "59447 ---> test5 ---> TEST5\n",
      "65966 ---> LÃ¼tcke ---> LA 1/4 TCKE\n",
      "83688 ---> Lajiness-O\\\\'Neill ---> LAJINESS-O\\\\'NEILL\n",
      "100078 ---> D\\\\'Souza ---> D\\\\'SOUZA\n",
      "101439 ---> Kate) ---> KATE)\n",
      "102493 ---> O\\\\'Boyle ---> O\\\\'BOYLE\n",
      "103415 ---> Vander_Zwan-Butler ---> VANDER_ZWAN-BUTLER\n",
      "154801 ---> Reed2 ---> REED2\n",
      "168078 ---> Ka`opua ---> KA`OPUA\n",
      "185579 ---> French\\tLiang Gao ---> FRENCH\\TLIANG GAO\n",
      "272251 ---> Hammon: Vicki ---> HAMMON: VICKI\n",
      "286004 ---> K.) Yeo ---> K) YEO\n",
      "292546 ---> Fisher/Gibson ---> FISHER/GIBSON\n",
      "335679 ---> Hutchens: Jane ---> HUTCHENS: JANE\n",
      "349715 ---> Peres: Deborah ---> PERES: DEBORAH\n",
      "415728 ---> Zahedi\" ---> ZAHEDI\"\n",
      "430584 ---> Fang\" ---> FANG\"\n",
      "462243 ---> Quarltere/Chiuchiolo ---> QUARLTERE/CHIUCHIOLO\n",
      "511376 ---> Al_Othman ---> AL_OTHMAN\n",
      "529927 ---> d&rsquo ---> D&RSQUO\n",
      "541829 ---> Tink\" Tinker\" ---> TINK\" TINKER\"\n",
      "547116 ---> Goldberg: Amy ---> GOLDBERG: AMY\n",
      "547175 ---> Rogers: Leanne ---> ROGERS: LEANNE\n",
      "557017 ---> D&rsquo ---> D&RSQUO\n",
      "587671 ---> Jennings: Alan ---> JENNINGS: ALAN\n",
      "595013 ---> Dutta: Ronald ---> DUTTA: RONALD\n",
      "595354 ---> Bove` ---> BOVE`\n",
      "619755 ---> F.) McKechnie ---> F) MCKECHNIE\n",
      "629617 ---> Alexander: Stephen ---> ALEXANDER: STEPHEN\n",
      "629724 ---> Sanders\" ---> SANDERS\"\n",
      "635770 ---> Randy\" Collins\" ---> RANDY\" COLLINS\"\n",
      "655197 ---> Faundez\" ---> FAUNDEZ\"\n",
      "656599 ---> Y.) Dong ---> Y) DONG\n",
      "656851 ---> Moore: ---> MOORE:\n",
      "694838 ---> Alexander: ---> ALEXANDER:\n",
      "696581 ---> Perrin\" ---> PERRIN\"\n",
      "700467 ---> Saadi6leray ---> SAADI6LERAY\n",
      "700697 ---> (bénai͏̈m) ---> BENAI[?]M\n",
      "705356 ---> Meimon-Mattei͏̈ ---> MEIMON-MATTEI[?]\n",
      "719557 ---> Dickerson: Jana ---> DICKERSON: JANA\n",
      "720060 ---> Basu: ---> BASU:\n",
      "720111 ---> Sanders: Duane ---> SANDERS: DUANE\n",
      "728663 ---> Arias_García ---> ARIAS_GARCIA\n",
      "793584 ---> try123456 ---> TRY123456\n",
      "795140 ---> engineer112 ---> ENGINEER112\n",
      "796169 ---> 1st Baron Acton ---> 1ST BARON ACTON\n",
      "797767 ---> Qiang/Pro. ---> QIANG/PRO\n",
      "808288 ---> Zo;omslas ---> ZO;OMSLAS\n",
      "811112 ---> (2012) ---> 2012\n",
      "813911 ---> A> ---> A>\n",
      "824685 ---> \"Gigi\" ---> \"GIGI\"\n",
      "829079 ---> Tao（陶凯） ---> TAO(Tao Kai )\n",
      "831793 ---> Reza) ---> REZA)\n",
      "832670 ---> DeSemple/Cowles ---> DESEMPLE/COWLES\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def print_dirty_lastname():\n",
    "    \"\"\"\"\"\"\n",
    "    result = []\n",
    "    char = set([\n",
    "        '\"', '&', '(', ')', '+', '.', '/', '0', '1', '2', '3', '4', '5', '6', ':', ';', '<', '>',\n",
    "        '?', '[', '\\\\', ']', '_', '`'])\n",
    "    for pid, l, ln in zip(ds_people.pid, ds_people.lastname, ds_people.lastname_norm):\n",
    "        if len(set(ln) & char) > 0:\n",
    "            print(pid, l, ln, sep=' ---> ')\n",
    "            result.append(pid)\n",
    "    return result\n",
    "\n",
    "dirty_ln_pid = print_dirty_lastname()\n",
    "len(dirty_ln_pid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(774733, 12)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_people = ds_people[\n",
    "    (~ds_people.pid.isin(dirty_ln_pid)) & (ds_people.lastname_norm.str.len() > 1)\n",
    "].copy().reset_index(drop=True)\n",
    "ds_people.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ORCID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===== AGVG75 =====\n",
      "===== A-8474-2013 =====\n",
      "===== J-5755-2014 =====\n",
      "===== https://www.ncbi.nlm.nih.gov/sites/myncbi/fletcher.white.1/bibliography/40234758/public/?sort=date&direction=descending =====\n",
      "===== Katherine M. Mathis =====\n",
      "===== <div itemscope itemtype=\"https://schema.org/Person\"><a itemprop=\"sameAs\" content=\"https://orcid.org/0000-0002-9254-5360\" href=\"https://orcid.org/0000-0002-9254-5360\" target=\"orcid.widget\" rel=\"noopener noreferrer\" style=\"vertical-align:top;\"><img src=\"htt =====\n",
      "===== SANTIAGOID =====\n",
      "===== 0002-4068-1168 =====\n",
      "===== B-5365-2009 =====\n",
      "===== 0000-0001-8507-256 =====\n",
      "===== C-4348-2013 =====\n",
      "===== my-orcid =====\n",
      "===== 0000-0001-6779-247 =====\n",
      "===== martipab =====\n"
     ]
    }
   ],
   "source": [
    "orcid_dict = {\n",
    "    '000-0003-2721-3770': '0000-0003-2721-3770',\n",
    "    '0000000246091337': '0000-0002-4609-1337',\n",
    "    '0000-00019801-2963': '0000-0001-9801-2963',\n",
    "    '00000003-2983-6330': '0000-0003-2983-6330',\n",
    "    '0000000348394400': '0000-0003-4839-4400',\n",
    "    '0000–0002–8587–4115': '0000-0002-8587-4115',\n",
    "    '0000000167270935': '0000-0001-6727-0935',\n",
    "}\n",
    "\n",
    "def clean_orcid(x):\n",
    "    \"\"\"\"\"\"\n",
    "    if x == '':\n",
    "        return x\n",
    "    xs = x.replace('\\u200b', '').strip(' \",“”')\n",
    "    for s in ['https://orcid.org/', 'http://orcid.org/', 'orcid.org/']:\n",
    "        if xs.startswith(s):\n",
    "            xs = xs[len(s):].strip()\n",
    "            break\n",
    "    parts = xs.split('-')\n",
    "    if len(xs) == 19 and len(parts) == 4 and all(len(e) == 4 for e in parts):\n",
    "        return xs\n",
    "    if xs in orcid_dict:\n",
    "        return orcid_dict[xs]\n",
    "    print('=====', xs, '=====')\n",
    "    return ''\n",
    "\n",
    "ds_people['orcid_norm'] = ds_people.orcid.apply(clean_orcid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1675, 1661)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ds_people.orcid != '').sum(), (ds_people.orcid_norm != '').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_people.to_hdf('results/people_df.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}