data/data_processing_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example data processing procedures (Negishi dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required modules\n",
    "import os\n",
    "import rdkit\n",
    "import re\n",
    "from rdkit import Chem\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading data from Reaxys&reg; export\n",
    "Assumes a query results set has been exported and saved as a file 'negishi.csv' in the same directory as this notebook. No data is included here for copyright purposes. See https://doi.org/10.26434/chemrxiv.12613214.v1 for a description of Reaxys&reg; exporting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>References</th>\n",
       "      <th>Links to Reaxys</th>\n",
       "      <th>Reaction: Links to Reaxys</th>\n",
       "      <th>Reaction</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>Patent; ZENTIVA, K.S.; RADL Stanislav; DOUBSKY...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...</td>\n",
       "      <td>60</td>\n",
       "      <td>0.52</td>\n",
       "      <td>palladium diacetate; ruphos</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>Patent; Gilead Sciences, Inc.; Korea Research ...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....</td>\n",
       "      <td>100</td>\n",
       "      <td>0.6</td>\n",
       "      <td>zinc</td>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>Patent; MEMORY PHARMACEUTICALS CORPORATION; WO...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1&gt;...</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>Article; Murphy-Benenato, Kerry E.; Gingipalli...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>Article; Greshock, Thomas J.; Moore, Keith P.;...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>((2-dicyclohexylphosphino-2',4?,6?-triisopropy...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11383</th>\n",
       "      <td>22895068</td>\n",
       "      <td>Patent; Shin-Etsu Chemical Co., Ltd.; US636927...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(=O)OC1=C(C=CC(I)=C1)C(F)(F)F.Cl[Zn]C=C&gt;&gt;CC(...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11384</th>\n",
       "      <td>22895069</td>\n",
       "      <td>Patent; Shin-Etsu Chemical Co., Ltd.; US636927...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>Cl[Zn]C=C.CC(C)(C)OC1=C(F)C(F)=C(I)C(F)=C1F&gt;&gt;C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11385</th>\n",
       "      <td>22895070</td>\n",
       "      <td>Patent; Shin-Etsu Chemical Co., Ltd.; US636927...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(C)(C)OC1=CC=C(F)C(I)=C1.Cl[Zn]C=C&gt;&gt;CC(C)(C)...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11386</th>\n",
       "      <td>22895072</td>\n",
       "      <td>Patent; Shin-Etsu Chemical Co., Ltd.; US636927...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>FC1=CC(OCC=C)=C(I)C=C1.CC(=C)[Zn]Cl&gt;&gt;CC(=C)C1=...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11387</th>\n",
       "      <td>22895073</td>\n",
       "      <td>Patent; Shin-Etsu Chemical Co., Ltd.; US636927...</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>https://www.reaxys.com/reaxys/secured/hopinto....</td>\n",
       "      <td>CC(=O)OC1=C(I)C(F)=C(F)C(F)=C1.CC(=C)[Zn]Cl&gt;&gt;C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11388 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Reaction ID                                         References  \\\n",
       "0         36844980  Patent; ZENTIVA, K.S.; RADL Stanislav; DOUBSKY...   \n",
       "1         28516312  Patent; Gilead Sciences, Inc.; Korea Research ...   \n",
       "2         24802995  Patent; MEMORY PHARMACEUTICALS CORPORATION; WO...   \n",
       "3         41263899  Article; Murphy-Benenato, Kerry E.; Gingipalli...   \n",
       "4         44324735  Article; Greshock, Thomas J.; Moore, Keith P.;...   \n",
       "...            ...                                                ...   \n",
       "11383     22895068  Patent; Shin-Etsu Chemical Co., Ltd.; US636927...   \n",
       "11384     22895069  Patent; Shin-Etsu Chemical Co., Ltd.; US636927...   \n",
       "11385     22895070  Patent; Shin-Etsu Chemical Co., Ltd.; US636927...   \n",
       "11386     22895072  Patent; Shin-Etsu Chemical Co., Ltd.; US636927...   \n",
       "11387     22895073  Patent; Shin-Etsu Chemical Co., Ltd.; US636927...   \n",
       "\n",
       "                                         Links to Reaxys  \\\n",
       "0      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "1      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "2      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "3      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "4      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "...                                                  ...   \n",
       "11383  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11384  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11385  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11386  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11387  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "\n",
       "                               Reaction: Links to Reaxys  \\\n",
       "0      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "1      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "2      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "3      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "4      https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "...                                                  ...   \n",
       "11383  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11384  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11385  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11386  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "11387  https://www.reaxys.com/reaxys/secured/hopinto....   \n",
       "\n",
       "                                                Reaction  \\\n",
       "0      CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...   \n",
       "1      CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....   \n",
       "2      COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1>...   \n",
       "3      Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...   \n",
       "4      CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...   \n",
       "...                                                  ...   \n",
       "11383  CC(=O)OC1=C(C=CC(I)=C1)C(F)(F)F.Cl[Zn]C=C>>CC(...   \n",
       "11384  Cl[Zn]C=C.CC(C)(C)OC1=C(F)C(F)=C(I)C(F)=C1F>>C...   \n",
       "11385  CC(C)(C)OC1=CC=C(F)C(I)=C1.Cl[Zn]C=C>>CC(C)(C)...   \n",
       "11386  FC1=CC(OCC=C)=C(I)C=C1.CC(=C)[Zn]Cl>>CC(=C)C1=...   \n",
       "11387  CC(=O)OC1=C(I)C(F)=C(F)C(F)=C1.CC(=C)[Zn]Cl>>C...   \n",
       "\n",
       "      Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0                                     60              0.52   \n",
       "1                                    100               0.6   \n",
       "2                                    100                 1   \n",
       "3                                     65                 2   \n",
       "4                                     50                 2   \n",
       "...                                  ...               ...   \n",
       "11383                                NaN               NaN   \n",
       "11384                                NaN               NaN   \n",
       "11385                                NaN               NaN   \n",
       "11386                                NaN               NaN   \n",
       "11387                                NaN               NaN   \n",
       "\n",
       "                                                 Reagent  \\\n",
       "0                            palladium diacetate; ruphos   \n",
       "1                                                   zinc   \n",
       "2                                                    NaN   \n",
       "3              tetrakis(triphenylphosphine) palladium(0)   \n",
       "4      ((2-dicyclohexylphosphino-2',4?,6?-triisopropy...   \n",
       "...                                                  ...   \n",
       "11383                                                NaN   \n",
       "11384                                                NaN   \n",
       "11385                                                NaN   \n",
       "11386                                                NaN   \n",
       "11387                                                NaN   \n",
       "\n",
       "                                        Catalyst Solvent (Reaction Details)  \n",
       "0                                            NaN            tetrahydrofuran  \n",
       "1      tris-(dibenzylideneacetone)dipalladium(0)     N,N-dimethyl-formamide  \n",
       "2      tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran  \n",
       "3                                            NaN            tetrahydrofuran  \n",
       "4                                            NaN            tetrahydrofuran  \n",
       "...                                          ...                        ...  \n",
       "11383                                        NaN                        NaN  \n",
       "11384                                        NaN                        NaN  \n",
       "11385                                        NaN                        NaN  \n",
       "11386                                        NaN                        NaN  \n",
       "11387                                        NaN                        NaN  \n",
       "\n",
       "[11388 rows x 10 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load dataset from Reaxys(R) csv export to pandas dataframe\n",
    "negishi_df = pd.read_csv('negishi.csv')\n",
    "\n",
    "# visualize dataframe\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Reaction ID',\n",
       " 'Reaction',\n",
       " 'Temperature (Reaction Details) [C]',\n",
       " 'Yield (numerical)',\n",
       " 'Reagent',\n",
       " 'Catalyst',\n",
       " 'Solvent (Reaction Details)']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trim dataframe to the desired columns\n",
    "negishi_df = negishi_df.drop(['References', 'Links to Reaxys', \n",
    "                              'Reaction: Links to Reaxys'], axis=1)\n",
    "\n",
    "# check the new dataframe columns\n",
    "negishi_df.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Reaction</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...</td>\n",
       "      <td>60</td>\n",
       "      <td>0.52</td>\n",
       "      <td>palladium diacetate; ruphos</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....</td>\n",
       "      <td>100</td>\n",
       "      <td>0.6</td>\n",
       "      <td>zinc</td>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1&gt;...</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>((2-dicyclohexylphosphino-2',4?,6?-triisopropy...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11383</th>\n",
       "      <td>22895068</td>\n",
       "      <td>CC(=O)OC1=C(C=CC(I)=C1)C(F)(F)F.Cl[Zn]C=C&gt;&gt;CC(...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11384</th>\n",
       "      <td>22895069</td>\n",
       "      <td>Cl[Zn]C=C.CC(C)(C)OC1=C(F)C(F)=C(I)C(F)=C1F&gt;&gt;C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11385</th>\n",
       "      <td>22895070</td>\n",
       "      <td>CC(C)(C)OC1=CC=C(F)C(I)=C1.Cl[Zn]C=C&gt;&gt;CC(C)(C)...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11386</th>\n",
       "      <td>22895072</td>\n",
       "      <td>FC1=CC(OCC=C)=C(I)C=C1.CC(=C)[Zn]Cl&gt;&gt;CC(=C)C1=...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11387</th>\n",
       "      <td>22895073</td>\n",
       "      <td>CC(=O)OC1=C(I)C(F)=C(F)C(F)=C1.CC(=C)[Zn]Cl&gt;&gt;C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11388 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Reaction ID                                           Reaction  \\\n",
       "0         36844980  CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...   \n",
       "1         28516312  CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....   \n",
       "2         24802995  COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1>...   \n",
       "3         41263899  Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...   \n",
       "4         44324735  CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...   \n",
       "...            ...                                                ...   \n",
       "11383     22895068  CC(=O)OC1=C(C=CC(I)=C1)C(F)(F)F.Cl[Zn]C=C>>CC(...   \n",
       "11384     22895069  Cl[Zn]C=C.CC(C)(C)OC1=C(F)C(F)=C(I)C(F)=C1F>>C...   \n",
       "11385     22895070  CC(C)(C)OC1=CC=C(F)C(I)=C1.Cl[Zn]C=C>>CC(C)(C)...   \n",
       "11386     22895072  FC1=CC(OCC=C)=C(I)C=C1.CC(=C)[Zn]Cl>>CC(=C)C1=...   \n",
       "11387     22895073  CC(=O)OC1=C(I)C(F)=C(F)C(F)=C1.CC(=C)[Zn]Cl>>C...   \n",
       "\n",
       "      Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0                                     60              0.52   \n",
       "1                                    100               0.6   \n",
       "2                                    100                 1   \n",
       "3                                     65                 2   \n",
       "4                                     50                 2   \n",
       "...                                  ...               ...   \n",
       "11383                                NaN               NaN   \n",
       "11384                                NaN               NaN   \n",
       "11385                                NaN               NaN   \n",
       "11386                                NaN               NaN   \n",
       "11387                                NaN               NaN   \n",
       "\n",
       "                                                 Reagent  \\\n",
       "0                            palladium diacetate; ruphos   \n",
       "1                                                   zinc   \n",
       "2                                                    NaN   \n",
       "3              tetrakis(triphenylphosphine) palladium(0)   \n",
       "4      ((2-dicyclohexylphosphino-2',4?,6?-triisopropy...   \n",
       "...                                                  ...   \n",
       "11383                                                NaN   \n",
       "11384                                                NaN   \n",
       "11385                                                NaN   \n",
       "11386                                                NaN   \n",
       "11387                                                NaN   \n",
       "\n",
       "                                        Catalyst Solvent (Reaction Details)  \n",
       "0                                            NaN            tetrahydrofuran  \n",
       "1      tris-(dibenzylideneacetone)dipalladium(0)     N,N-dimethyl-formamide  \n",
       "2      tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran  \n",
       "3                                            NaN            tetrahydrofuran  \n",
       "4                                            NaN            tetrahydrofuran  \n",
       "...                                          ...                        ...  \n",
       "11383                                        NaN                        NaN  \n",
       "11384                                        NaN                        NaN  \n",
       "11385                                        NaN                        NaN  \n",
       "11386                                        NaN                        NaN  \n",
       "11387                                        NaN                        NaN  \n",
       "\n",
       "[11388 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualize the new dataframe\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trim dataset where required values are missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 11388 entries, 0 to 11387\n",
      "Data columns (total 7 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         11388 non-null  int64 \n",
      " 1   Reaction                            11355 non-null  object\n",
      " 2   Temperature (Reaction Details) [C]  9676 non-null   object\n",
      " 3   Yield (numerical)                   7685 non-null   object\n",
      " 4   Reagent                             8719 non-null   object\n",
      " 5   Catalyst                            3332 non-null   object\n",
      " 6   Solvent (Reaction Details)          10739 non-null  object\n",
      "dtypes: int64(1), object(6)\n",
      "memory usage: 622.9+ KB\n"
     ]
    }
   ],
   "source": [
    "# get overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7406 entries, 0 to 7684\n",
      "Data columns (total 7 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         7406 non-null   int64 \n",
      " 1   Reaction                            7406 non-null   object\n",
      " 2   Temperature (Reaction Details) [C]  6592 non-null   object\n",
      " 3   Yield (numerical)                   7406 non-null   object\n",
      " 4   Reagent                             5975 non-null   object\n",
      " 5   Catalyst                            1882 non-null   object\n",
      " 6   Solvent (Reaction Details)          7406 non-null   object\n",
      "dtypes: int64(1), object(6)\n",
      "memory usage: 462.9+ KB\n"
     ]
    }
   ],
   "source": [
    "# remove reactions without reported yields, solvent, or reaction encoding\n",
    "# reactions without reagents or catalysts were retained in this dataset as\n",
    "# there are known examples where, e.g., heating is sufficient for reactivity\n",
    "negishi_df = negishi_df[negishi_df['Yield (numerical)'].notna()]\n",
    "negishi_df = negishi_df[negishi_df['Solvent (Reaction Details)'].notna()]\n",
    "negishi_df = negishi_df[negishi_df['Reaction'].notna()]\n",
    "\n",
    "# get overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Reaction</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...</td>\n",
       "      <td>60</td>\n",
       "      <td>0.52</td>\n",
       "      <td>palladium diacetate; ruphos</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....</td>\n",
       "      <td>100</td>\n",
       "      <td>0.6</td>\n",
       "      <td>zinc</td>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1&gt;...</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>((2-dicyclohexylphosphino-2',4?,6?-triisopropy...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>57; 19</td>\n",
       "      <td>1,3-bis[(diphenylphosphino)propane]dichloronic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...</td>\n",
       "      <td>20</td>\n",
       "      <td>57; 33</td>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>NaN</td>\n",
       "      <td>toluene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...</td>\n",
       "      <td>25</td>\n",
       "      <td>6; 83</td>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride;...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...</td>\n",
       "      <td>23</td>\n",
       "      <td>6; 84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cl2Pd(DPEphos)</td>\n",
       "      <td>toluene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....</td>\n",
       "      <td>50</td>\n",
       "      <td>71; 15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID                                           Reaction  \\\n",
       "0        36844980  CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...   \n",
       "1        28516312  CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....   \n",
       "2        24802995  COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1>...   \n",
       "3        41263899  Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...   \n",
       "4        44324735  CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...   \n",
       "...           ...                                                ...   \n",
       "7680      4559574  CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...   \n",
       "7681     42535671  CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...   \n",
       "7682      2446129  CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...   \n",
       "7683      9515318  CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...   \n",
       "7684      9407346  ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....   \n",
       "\n",
       "     Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0                                    60              0.52   \n",
       "1                                   100               0.6   \n",
       "2                                   100                 1   \n",
       "3                                    65                 2   \n",
       "4                                    50                 2   \n",
       "...                                 ...               ...   \n",
       "7680                                NaN            57; 19   \n",
       "7681                                 20            57; 33   \n",
       "7682                                 25             6; 83   \n",
       "7683                                 23             6; 84   \n",
       "7684                                 50            71; 15   \n",
       "\n",
       "                                                Reagent  \\\n",
       "0                           palladium diacetate; ruphos   \n",
       "1                                                  zinc   \n",
       "2                                                   NaN   \n",
       "3             tetrakis(triphenylphosphine) palladium(0)   \n",
       "4     ((2-dicyclohexylphosphino-2',4?,6?-triisopropy...   \n",
       "...                                                 ...   \n",
       "7680  1,3-bis[(diphenylphosphino)propane]dichloronic...   \n",
       "7681      bis-triphenylphosphine-palladium(II) chloride   \n",
       "7682  bis-triphenylphosphine-palladium(II) chloride;...   \n",
       "7683                                                NaN   \n",
       "7684                                                NaN   \n",
       "\n",
       "                                       Catalyst Solvent (Reaction Details)  \n",
       "0                                           NaN            tetrahydrofuran  \n",
       "1     tris-(dibenzylideneacetone)dipalladium(0)     N,N-dimethyl-formamide  \n",
       "2     tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran  \n",
       "3                                           NaN            tetrahydrofuran  \n",
       "4                                           NaN            tetrahydrofuran  \n",
       "...                                         ...                        ...  \n",
       "7680                                        NaN            tetrahydrofuran  \n",
       "7681                                        NaN                    toluene  \n",
       "7682                                        NaN     N,N-dimethyl-formamide  \n",
       "7683                             Cl2Pd(DPEphos)                    toluene  \n",
       "7684  tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran  \n",
       "\n",
       "[7406 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualize current dataframe\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate starting materials and products from the reaction SMILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>60</td>\n",
       "      <td>0.52</td>\n",
       "      <td>palladium diacetate; ruphos</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...</td>\n",
       "      <td>CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>100</td>\n",
       "      <td>0.6</td>\n",
       "      <td>zinc</td>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1</td>\n",
       "      <td>COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...</td>\n",
       "      <td>CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>((2-dicyclohexylphosphino-2',4?,6?-triisopropy...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...</td>\n",
       "      <td>O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>NaN</td>\n",
       "      <td>57; 19</td>\n",
       "      <td>1,3-bis[(diphenylphosphino)propane]dichloronic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...</td>\n",
       "      <td>CC(C)CCC1=NC(C)=CN(=O)=C1C.CC(C)CCC1=C(C)N=CC(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>20</td>\n",
       "      <td>57; 33</td>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>NaN</td>\n",
       "      <td>toluene</td>\n",
       "      <td>CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...</td>\n",
       "      <td>CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>25</td>\n",
       "      <td>6; 83</td>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride;...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "      <td>CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...</td>\n",
       "      <td>CCCC\\C=C\\C=C\\CCCC.CCCC\\C=C\\C1=CCCCC1=O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>23</td>\n",
       "      <td>6; 84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cl2Pd(DPEphos)</td>\n",
       "      <td>toluene</td>\n",
       "      <td>CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...</td>\n",
       "      <td>CCCCCCC#CC1=NC=CS1.CCCCCC\\C=C(Br)\\C1=NC=CS1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>50</td>\n",
       "      <td>71; 15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....</td>\n",
       "      <td>CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1.CC1=NC2=C(N...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                                 60              0.52   \n",
       "1        28516312                                100               0.6   \n",
       "2        24802995                                100                 1   \n",
       "3        41263899                                 65                 2   \n",
       "4        44324735                                 50                 2   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                                NaN            57; 19   \n",
       "7681     42535671                                 20            57; 33   \n",
       "7682      2446129                                 25             6; 83   \n",
       "7683      9515318                                 23             6; 84   \n",
       "7684      9407346                                 50            71; 15   \n",
       "\n",
       "                                                Reagent  \\\n",
       "0                           palladium diacetate; ruphos   \n",
       "1                                                  zinc   \n",
       "2                                                   NaN   \n",
       "3             tetrakis(triphenylphosphine) palladium(0)   \n",
       "4     ((2-dicyclohexylphosphino-2',4?,6?-triisopropy...   \n",
       "...                                                 ...   \n",
       "7680  1,3-bis[(diphenylphosphino)propane]dichloronic...   \n",
       "7681      bis-triphenylphosphine-palladium(II) chloride   \n",
       "7682  bis-triphenylphosphine-palladium(II) chloride;...   \n",
       "7683                                                NaN   \n",
       "7684                                                NaN   \n",
       "\n",
       "                                       Catalyst Solvent (Reaction Details)  \\\n",
       "0                                           NaN            tetrahydrofuran   \n",
       "1     tris-(dibenzylideneacetone)dipalladium(0)     N,N-dimethyl-formamide   \n",
       "2     tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran   \n",
       "3                                           NaN            tetrahydrofuran   \n",
       "4                                           NaN            tetrahydrofuran   \n",
       "...                                         ...                        ...   \n",
       "7680                                        NaN            tetrahydrofuran   \n",
       "7681                                        NaN                    toluene   \n",
       "7682                                        NaN     N,N-dimethyl-formamide   \n",
       "7683                             Cl2Pd(DPEphos)                    toluene   \n",
       "7684  tetrakis(triphenylphosphine) palladium(0)            tetrahydrofuran   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...   \n",
       "1     CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....   \n",
       "2         COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1   \n",
       "3     Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...   \n",
       "4     CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...   \n",
       "...                                                 ...   \n",
       "7680  CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...   \n",
       "7681  CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...   \n",
       "7682  CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...   \n",
       "7683  CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...   \n",
       "7684  ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....   \n",
       "\n",
       "                                               products  \n",
       "0     CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)(...  \n",
       "1        CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N  \n",
       "2                  COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1  \n",
       "3     CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC=...  \n",
       "4     O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C=...  \n",
       "...                                                 ...  \n",
       "7680  CC(C)CCC1=NC(C)=CN(=O)=C1C.CC(C)CCC1=C(C)N=CC(...  \n",
       "7681  CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[N...  \n",
       "7682             CCCC\\C=C\\C=C\\CCCC.CCCC\\C=C\\C1=CCCCC1=O  \n",
       "7683        CCCCCCC#CC1=NC=CS1.CCCCCC\\C=C(Br)\\C1=NC=CS1  \n",
       "7684  CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1.CC1=NC2=C(N...  \n",
       "\n",
       "[7406 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split 'Reaction' into 'reactants' and 'products' by '>>' and\n",
    "# drop 'Reaction' column\n",
    "negishi_df[['reactants', 'products']] = negishi_df['Reaction'].str.split('>>',\n",
    "                                                                         expand=True)\n",
    "negishi_df = negishi_df.drop(['Reaction'], axis=1)\n",
    "\n",
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7406 entries, 0 to 7684\n",
      "Data columns (total 8 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         7406 non-null   int64 \n",
      " 1   Temperature (Reaction Details) [C]  6592 non-null   object\n",
      " 2   Yield (numerical)                   7406 non-null   object\n",
      " 3   Reagent                             5975 non-null   object\n",
      " 4   Catalyst                            1882 non-null   object\n",
      " 5   Solvent (Reaction Details)          7406 non-null   object\n",
      " 6   reactants                           7406 non-null   object\n",
      " 7   products                            7406 non-null   object\n",
      "dtypes: int64(1), object(7)\n",
      "memory usage: 520.7+ KB\n"
     ]
    }
   ],
   "source": [
    "# check overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate multiple components of each reaction condition type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split 'Temperature (Reaction Details) [C]', 'Yield (numerical)', 'Reagent',\n",
    "# 'Catalyst', 'Solvent (Reaction Details)' to lists by ';' in place\n",
    "col_splits = ['Yield (numerical)', 'Reagent',\n",
    "          'Catalyst', 'Solvent (Reaction Details)']\n",
    "for col in col_splits:\n",
    "    negishi_df[[col]] = negishi_df[col].str.split(';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>60</td>\n",
       "      <td>[0.52]</td>\n",
       "      <td>[palladium diacetate,  ruphos]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...</td>\n",
       "      <td>CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>100</td>\n",
       "      <td>[0.6]</td>\n",
       "      <td>[zinc]</td>\n",
       "      <td>[tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....</td>\n",
       "      <td>CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>100</td>\n",
       "      <td>[1]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1</td>\n",
       "      <td>COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>65</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...</td>\n",
       "      <td>CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>50</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[((2-dicyclohexylphosphino-2',4?,6?-triisoprop...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...</td>\n",
       "      <td>O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[57,  19]</td>\n",
       "      <td>[1,3-bis[(diphenylphosphino)propane]dichloroni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...</td>\n",
       "      <td>CC(C)CCC1=NC(C)=CN(=O)=C1C.CC(C)CCC1=C(C)N=CC(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>20</td>\n",
       "      <td>[57,  33]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...</td>\n",
       "      <td>CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>25</td>\n",
       "      <td>[6,  83]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...</td>\n",
       "      <td>CCCC\\C=C\\C=C\\CCCC.CCCC\\C=C\\C1=CCCCC1=O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>23</td>\n",
       "      <td>[6,  84]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Cl2Pd(DPEphos)]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...</td>\n",
       "      <td>CCCCCCC#CC1=NC=CS1.CCCCCC\\C=C(Br)\\C1=NC=CS1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>50</td>\n",
       "      <td>[71,  15]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....</td>\n",
       "      <td>CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1.CC1=NC2=C(N...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                                 60            [0.52]   \n",
       "1        28516312                                100             [0.6]   \n",
       "2        24802995                                100               [1]   \n",
       "3        41263899                                 65               [2]   \n",
       "4        44324735                                 50               [2]   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                                NaN         [57,  19]   \n",
       "7681     42535671                                 20         [57,  33]   \n",
       "7682      2446129                                 25          [6,  83]   \n",
       "7683      9515318                                 23          [6,  84]   \n",
       "7684      9407346                                 50         [71,  15]   \n",
       "\n",
       "                                                Reagent  \\\n",
       "0                        [palladium diacetate,  ruphos]   \n",
       "1                                                [zinc]   \n",
       "2                                                   NaN   \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]   \n",
       "4     [((2-dicyclohexylphosphino-2',4?,6?-triisoprop...   \n",
       "...                                                 ...   \n",
       "7680  [1,3-bis[(diphenylphosphino)propane]dichloroni...   \n",
       "7681    [bis-triphenylphosphine-palladium(II) chloride]   \n",
       "7682  [bis-triphenylphosphine-palladium(II) chloride...   \n",
       "7683                                                NaN   \n",
       "7684                                                NaN   \n",
       "\n",
       "                                         Catalyst Solvent (Reaction Details)  \\\n",
       "0                                             NaN          [tetrahydrofuran]   \n",
       "1     [tris-(dibenzylideneacetone)dipalladium(0)]   [N,N-dimethyl-formamide]   \n",
       "2     [tetrakis(triphenylphosphine) palladium(0)]          [tetrahydrofuran]   \n",
       "3                                             NaN          [tetrahydrofuran]   \n",
       "4                                             NaN          [tetrahydrofuran]   \n",
       "...                                           ...                        ...   \n",
       "7680                                          NaN          [tetrahydrofuran]   \n",
       "7681                                          NaN                  [toluene]   \n",
       "7682                                          NaN   [N,N-dimethyl-formamide]   \n",
       "7683                             [Cl2Pd(DPEphos)]                  [toluene]   \n",
       "7684  [tetrakis(triphenylphosphine) palladium(0)]          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)O...   \n",
       "1     CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1....   \n",
       "2         COC1=C(OC)C=C2C(Br)=CN=NC2=C1.Br[Zn]C1=NC=CS1   \n",
       "3     Br[Zn]C1=CC=CN=C1.CSC1=NC(=NC2=NC(N)=C(C#N)C(N...   \n",
       "4     CC(C)(C)C([O-])=O.[Zn+]CC1=NC=C(CN2CCOCC2)C=C1...   \n",
       "...                                                 ...   \n",
       "7680  CC1=CN(=O)=C(C)C(Cl)=N1.CC(C)CC[Zn]CCC(C)C.CC1...   \n",
       "7681  CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F)...   \n",
       "7682  CCCC\\C=C\\[Zn]\\C=C\\CCCC.FC(F)(F)S(=O)(=O)OC1=CC...   \n",
       "7683  CCCCCC\\C=C(/Br)Br.Br[Zn]C1=NC=CS1.CCCCCC\\C=C(/...   \n",
       "7684  ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1.C[Zn]Br....   \n",
       "\n",
       "                                               products  \n",
       "0     CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)(...  \n",
       "1        CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N  \n",
       "2                  COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1  \n",
       "3     CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC=...  \n",
       "4     O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C=...  \n",
       "...                                                 ...  \n",
       "7680  CC(C)CCC1=NC(C)=CN(=O)=C1C.CC(C)CCC1=C(C)N=CC(...  \n",
       "7681  CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[N...  \n",
       "7682             CCCC\\C=C\\C=C\\CCCC.CCCC\\C=C\\C1=CCCCC1=O  \n",
       "7683        CCCCCCC#CC1=NC=CS1.CCCCCC\\C=C(Br)\\C1=NC=CS1  \n",
       "7684  CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1.CC1=NC2=C(N...  \n",
       "\n",
       "[7406 rows x 8 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate individual molecules in the reaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Reagent</th>\n",
       "      <th>Catalyst</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>60</td>\n",
       "      <td>[0.52]</td>\n",
       "      <td>[palladium diacetate,  ruphos]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...</td>\n",
       "      <td>[CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>100</td>\n",
       "      <td>[0.6]</td>\n",
       "      <td>[zinc]</td>\n",
       "      <td>[tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>100</td>\n",
       "      <td>[1]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>65</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...</td>\n",
       "      <td>[CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>50</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[((2-dicyclohexylphosphino-2',4?,6?-triisoprop...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...</td>\n",
       "      <td>[O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[57,  19]</td>\n",
       "      <td>[1,3-bis[(diphenylphosphino)propane]dichloroni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...</td>\n",
       "      <td>[CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>20</td>\n",
       "      <td>[57,  33]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>25</td>\n",
       "      <td>[6,  83]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...</td>\n",
       "      <td>[CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>23</td>\n",
       "      <td>[6,  84]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Cl2Pd(DPEphos)]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...</td>\n",
       "      <td>[CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>50</td>\n",
       "      <td>[71,  15]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...</td>\n",
       "      <td>[CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                                 60            [0.52]   \n",
       "1        28516312                                100             [0.6]   \n",
       "2        24802995                                100               [1]   \n",
       "3        41263899                                 65               [2]   \n",
       "4        44324735                                 50               [2]   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                                NaN         [57,  19]   \n",
       "7681     42535671                                 20         [57,  33]   \n",
       "7682      2446129                                 25          [6,  83]   \n",
       "7683      9515318                                 23          [6,  84]   \n",
       "7684      9407346                                 50         [71,  15]   \n",
       "\n",
       "                                                Reagent  \\\n",
       "0                        [palladium diacetate,  ruphos]   \n",
       "1                                                [zinc]   \n",
       "2                                                   NaN   \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]   \n",
       "4     [((2-dicyclohexylphosphino-2',4?,6?-triisoprop...   \n",
       "...                                                 ...   \n",
       "7680  [1,3-bis[(diphenylphosphino)propane]dichloroni...   \n",
       "7681    [bis-triphenylphosphine-palladium(II) chloride]   \n",
       "7682  [bis-triphenylphosphine-palladium(II) chloride...   \n",
       "7683                                                NaN   \n",
       "7684                                                NaN   \n",
       "\n",
       "                                         Catalyst Solvent (Reaction Details)  \\\n",
       "0                                             NaN          [tetrahydrofuran]   \n",
       "1     [tris-(dibenzylideneacetone)dipalladium(0)]   [N,N-dimethyl-formamide]   \n",
       "2     [tetrakis(triphenylphosphine) palladium(0)]          [tetrahydrofuran]   \n",
       "3                                             NaN          [tetrahydrofuran]   \n",
       "4                                             NaN          [tetrahydrofuran]   \n",
       "...                                           ...                        ...   \n",
       "7680                                          NaN          [tetrahydrofuran]   \n",
       "7681                                          NaN                  [toluene]   \n",
       "7682                                          NaN   [N,N-dimethyl-formamide]   \n",
       "7683                             [Cl2Pd(DPEphos)]                  [toluene]   \n",
       "7684  [tetrakis(triphenylphosphine) palladium(0)]          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     [CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...   \n",
       "1     [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...   \n",
       "2      [COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]   \n",
       "3     [Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...   \n",
       "4     [CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...   \n",
       "...                                                 ...   \n",
       "7680  [CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...   \n",
       "7682  [CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...   \n",
       "7683  [CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...   \n",
       "7684  [ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...   \n",
       "\n",
       "                                               products  \n",
       "0     [CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...  \n",
       "1      [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]  \n",
       "2                [COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]  \n",
       "3     [CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...  \n",
       "4     [O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...  \n",
       "...                                                 ...  \n",
       "7680  [CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...  \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...  \n",
       "7682          [CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]  \n",
       "7683     [CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]  \n",
       "7684  [CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...  \n",
       "\n",
       "[7406 rows x 8 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split 'reactants' and 'products' into lists by '.' in place\n",
    "col_splits_2 = ['reactants', 'products']\n",
    "for col in col_splits_2:\n",
    "    negishi_df[[col]] = negishi_df[col].str.split('.')\n",
    "    \n",
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine all added agents into single reagents field\n",
    "Reagents and catalysts are sometimes incorrectly recorded in Reaxys&reg; results, so we combine them here to avoid conflicts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "      <th>reagents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>60</td>\n",
       "      <td>[0.52]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...</td>\n",
       "      <td>[CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...</td>\n",
       "      <td>[palladium diacetate,  ruphos]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>100</td>\n",
       "      <td>[0.6]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]</td>\n",
       "      <td>[zinc, tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>100</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>65</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...</td>\n",
       "      <td>[CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>50</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...</td>\n",
       "      <td>[O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...</td>\n",
       "      <td>[((2-dicyclohexylphosphino-2',4?,6?-triisoprop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[57,  19]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...</td>\n",
       "      <td>[CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...</td>\n",
       "      <td>[1,3-bis[(diphenylphosphino)propane]dichloroni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>20</td>\n",
       "      <td>[57,  33]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>25</td>\n",
       "      <td>[6,  83]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...</td>\n",
       "      <td>[CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>23</td>\n",
       "      <td>[6,  84]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...</td>\n",
       "      <td>[CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]</td>\n",
       "      <td>[Cl2Pd(DPEphos)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>50</td>\n",
       "      <td>[71,  15]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...</td>\n",
       "      <td>[CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                                 60            [0.52]   \n",
       "1        28516312                                100             [0.6]   \n",
       "2        24802995                                100               [1]   \n",
       "3        41263899                                 65               [2]   \n",
       "4        44324735                                 50               [2]   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                                NaN         [57,  19]   \n",
       "7681     42535671                                 20         [57,  33]   \n",
       "7682      2446129                                 25          [6,  83]   \n",
       "7683      9515318                                 23          [6,  84]   \n",
       "7684      9407346                                 50         [71,  15]   \n",
       "\n",
       "     Solvent (Reaction Details)  \\\n",
       "0             [tetrahydrofuran]   \n",
       "1      [N,N-dimethyl-formamide]   \n",
       "2             [tetrahydrofuran]   \n",
       "3             [tetrahydrofuran]   \n",
       "4             [tetrahydrofuran]   \n",
       "...                         ...   \n",
       "7680          [tetrahydrofuran]   \n",
       "7681                  [toluene]   \n",
       "7682   [N,N-dimethyl-formamide]   \n",
       "7683                  [toluene]   \n",
       "7684          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     [CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...   \n",
       "1     [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...   \n",
       "2      [COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]   \n",
       "3     [Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...   \n",
       "4     [CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...   \n",
       "...                                                 ...   \n",
       "7680  [CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...   \n",
       "7682  [CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...   \n",
       "7683  [CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...   \n",
       "7684  [ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...   \n",
       "\n",
       "                                               products  \\\n",
       "0     [CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...   \n",
       "1      [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]   \n",
       "2                [COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]   \n",
       "3     [CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...   \n",
       "4     [O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...   \n",
       "...                                                 ...   \n",
       "7680  [CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...   \n",
       "7682          [CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]   \n",
       "7683     [CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]   \n",
       "7684  [CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...   \n",
       "\n",
       "                                               reagents  \n",
       "0                        [palladium diacetate,  ruphos]  \n",
       "1     [zinc, tris-(dibenzylideneacetone)dipalladium(0)]  \n",
       "2           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "4     [((2-dicyclohexylphosphino-2',4?,6?-triisoprop...  \n",
       "...                                                 ...  \n",
       "7680  [1,3-bis[(diphenylphosphino)propane]dichloroni...  \n",
       "7681    [bis-triphenylphosphine-palladium(II) chloride]  \n",
       "7682  [bis-triphenylphosphine-palladium(II) chloride...  \n",
       "7683                                   [Cl2Pd(DPEphos)]  \n",
       "7684        [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "\n",
       "[7406 rows x 7 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate 'Reagent' and 'Catalyst' columns to single 'reagents' lists\n",
    "# first replace NaN values with empty lists\n",
    "r_na = negishi_df['Reagent'].isna()\n",
    "c_na = negishi_df['Catalyst'].isna()\n",
    "negishi_df.loc[r_na, 'Reagent'] = pd.Series([[]] * r_na.sum()).values\n",
    "negishi_df.loc[c_na, 'Catalyst'] = pd.Series([[]] * c_na.sum()).values\n",
    "\n",
    "# concatenate lists and drop original columns\n",
    "negishi_df['reagents'] = negishi_df['Reagent'] + negishi_df['Catalyst']\n",
    "negishi_df = negishi_df.drop(['Reagent', 'Catalyst'], axis=1)\n",
    "\n",
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7406 entries, 0 to 7684\n",
      "Data columns (total 7 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         7406 non-null   int64 \n",
      " 1   Temperature (Reaction Details) [C]  6592 non-null   object\n",
      " 2   Yield (numerical)                   7406 non-null   object\n",
      " 3   Solvent (Reaction Details)          7406 non-null   object\n",
      " 4   reactants                           7406 non-null   object\n",
      " 5   products                            7406 non-null   object\n",
      " 6   reagents                            7406 non-null   object\n",
      "dtypes: int64(1), object(6)\n",
      "memory usage: 462.9+ KB\n"
     ]
    }
   ],
   "source": [
    "# get overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fill in missing temperatures, if appropriate\n",
    "As seen in the negishi_df.info table above, the temperature column is fairly well recorded. We make the assumption that missing values are room temperature (20 °C), such that this field can be retained. This is not the case with all datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    20\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# check most common (i.e., room) temperature to replace NaN values\n",
    "print(negishi_df['Temperature (Reaction Details) [C]'].mode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "      <th>reagents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>60</td>\n",
       "      <td>[0.52]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...</td>\n",
       "      <td>[CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...</td>\n",
       "      <td>[palladium diacetate,  ruphos]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>100</td>\n",
       "      <td>[0.6]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]</td>\n",
       "      <td>[zinc, tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>100</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>65</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...</td>\n",
       "      <td>[CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>50</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...</td>\n",
       "      <td>[O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...</td>\n",
       "      <td>[((2-dicyclohexylphosphino-2',4?,6?-triisoprop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>20</td>\n",
       "      <td>[57,  19]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...</td>\n",
       "      <td>[CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...</td>\n",
       "      <td>[1,3-bis[(diphenylphosphino)propane]dichloroni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>20</td>\n",
       "      <td>[57,  33]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>25</td>\n",
       "      <td>[6,  83]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...</td>\n",
       "      <td>[CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>23</td>\n",
       "      <td>[6,  84]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...</td>\n",
       "      <td>[CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]</td>\n",
       "      <td>[Cl2Pd(DPEphos)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>50</td>\n",
       "      <td>[71,  15]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...</td>\n",
       "      <td>[CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                                 60            [0.52]   \n",
       "1        28516312                                100             [0.6]   \n",
       "2        24802995                                100               [1]   \n",
       "3        41263899                                 65               [2]   \n",
       "4        44324735                                 50               [2]   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                                 20         [57,  19]   \n",
       "7681     42535671                                 20         [57,  33]   \n",
       "7682      2446129                                 25          [6,  83]   \n",
       "7683      9515318                                 23          [6,  84]   \n",
       "7684      9407346                                 50         [71,  15]   \n",
       "\n",
       "     Solvent (Reaction Details)  \\\n",
       "0             [tetrahydrofuran]   \n",
       "1      [N,N-dimethyl-formamide]   \n",
       "2             [tetrahydrofuran]   \n",
       "3             [tetrahydrofuran]   \n",
       "4             [tetrahydrofuran]   \n",
       "...                         ...   \n",
       "7680          [tetrahydrofuran]   \n",
       "7681                  [toluene]   \n",
       "7682   [N,N-dimethyl-formamide]   \n",
       "7683                  [toluene]   \n",
       "7684          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     [CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...   \n",
       "1     [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(Br)=CC(C)=C1...   \n",
       "2      [COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]   \n",
       "3     [Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...   \n",
       "4     [CC(C)(C)C([O-])=O, [Zn+]CC1=NC=C(CN2CCOCC2)C=...   \n",
       "...                                                 ...   \n",
       "7680  [CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C, ...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...   \n",
       "7682  [CCCC\\C=C\\[Zn]\\C=C\\CCCC, FC(F)(F)S(=O)(=O)OC1=...   \n",
       "7683  [CCCCCC\\C=C(/Br)Br, Br[Zn]C1=NC=CS1, CCCCCC\\C=...   \n",
       "7684  [ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1, C[Zn]B...   \n",
       "\n",
       "                                               products  \\\n",
       "0     [CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...   \n",
       "1      [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]   \n",
       "2                [COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]   \n",
       "3     [CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...   \n",
       "4     [O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...   \n",
       "...                                                 ...   \n",
       "7680  [CC(C)CCC1=NC(C)=CN(=O)=C1C, CC(C)CCC1=C(C)N=C...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(C)C(Br)=C(C)N3[B](F)(F)[...   \n",
       "7682          [CCCC\\C=C\\C=C\\CCCC, CCCC\\C=C\\C1=CCCCC1=O]   \n",
       "7683     [CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]   \n",
       "7684  [CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...   \n",
       "\n",
       "                                               reagents  \n",
       "0                        [palladium diacetate,  ruphos]  \n",
       "1     [zinc, tris-(dibenzylideneacetone)dipalladium(0)]  \n",
       "2           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "4     [((2-dicyclohexylphosphino-2',4?,6?-triisoprop...  \n",
       "...                                                 ...  \n",
       "7680  [1,3-bis[(diphenylphosphino)propane]dichloroni...  \n",
       "7681    [bis-triphenylphosphine-palladium(II) chloride]  \n",
       "7682  [bis-triphenylphosphine-palladium(II) chloride...  \n",
       "7683                                   [Cl2Pd(DPEphos)]  \n",
       "7684        [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "\n",
       "[7406 rows x 7 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fill in NaN temperatures with room temperature (20)\n",
    "t_na = negishi_df['Temperature (Reaction Details) [C]'].isna()\n",
    "negishi_df.loc[t_na, 'Temperature (Reaction Details) [C]'] = pd.Series([20] * t_na.sum()).values\n",
    "\n",
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 7406 entries, 0 to 7684\n",
      "Data columns (total 7 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         7406 non-null   int64 \n",
      " 1   Temperature (Reaction Details) [C]  7406 non-null   object\n",
      " 2   Yield (numerical)                   7406 non-null   object\n",
      " 3   Solvent (Reaction Details)          7406 non-null   object\n",
      " 4   reactants                           7406 non-null   object\n",
      " 5   products                            7406 non-null   object\n",
      " 6   reagents                            7406 non-null   object\n",
      "dtypes: int64(1), object(6)\n",
      "memory usage: 462.9+ KB\n"
     ]
    }
   ],
   "source": [
    "# get overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean instances of repeated reagent entries\n",
    "It is common to find many reactants, reagents, solvents, temperatures, etc. repeated several times in a reaction entry. It is unclear if this represents multiple equivalents of a certain compound, multiple reaction stages, or otherwise, so we filter the duplicates here such that only unique values remain.<br>\n",
    "\n",
    "This requires some trial and error to effectively split the text fields to only include the information of interest. The temperature field can be particularly challenging due to the various symbols involved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Yield (numerical)</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "      <th>reagents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>[60]</td>\n",
       "      <td>[0.52]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...</td>\n",
       "      <td>[CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...</td>\n",
       "      <td>[palladium diacetate, ruphos]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>[100]</td>\n",
       "      <td>[0.6]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[N#C[Zn]C#N, CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(...</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]</td>\n",
       "      <td>[zinc, tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>[100]</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>[65]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...</td>\n",
       "      <td>[CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44324735</td>\n",
       "      <td>[50]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[BrC1=CC=C2NC=C(C(=O)[C@@H]3CCCN3C(=O)OCC3=CC=...</td>\n",
       "      <td>[O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...</td>\n",
       "      <td>[((2-dicyclohexylphosphino-2',4?,6?-triisoprop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7680</th>\n",
       "      <td>4559574</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[57,  19]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C]</td>\n",
       "      <td>[CC(C)CCC1=C(C)N=CC(C)=N1, CC(C)CCC1=NC(C)=CN(...</td>\n",
       "      <td>[1,3-bis[(diphenylphosphino)propane]dichloroni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7681</th>\n",
       "      <td>42535671</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[57,  33]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...</td>\n",
       "      <td>[CCC1=C(C)C2=C(C)C3=C(C)C(C)=C(C)N3[B](F)(F)[N...</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7682</th>\n",
       "      <td>2446129</td>\n",
       "      <td>[25]</td>\n",
       "      <td>[6,  83]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[FC(F)(F)S(=O)(=O)OC1=CCCCC1=O, CCCC\\C=C\\[Zn]\\...</td>\n",
       "      <td>[CCCC\\C=C\\C1=CCCCC1=O, CCCC\\C=C\\C=C\\CCCC]</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7683</th>\n",
       "      <td>9515318</td>\n",
       "      <td>[23]</td>\n",
       "      <td>[6,  84]</td>\n",
       "      <td>[toluene]</td>\n",
       "      <td>[Br[Zn]C1=NC=CS1, CCCCCC\\C=C(/Br)Br]</td>\n",
       "      <td>[CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]</td>\n",
       "      <td>[Cl2Pd(DPEphos)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7684</th>\n",
       "      <td>9407346</td>\n",
       "      <td>[50]</td>\n",
       "      <td>[71,  15]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[C[Zn]Br, ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1]</td>\n",
       "      <td>[CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C] Yield (numerical)  \\\n",
       "0        36844980                               [60]            [0.52]   \n",
       "1        28516312                              [100]             [0.6]   \n",
       "2        24802995                              [100]               [1]   \n",
       "3        41263899                               [65]               [2]   \n",
       "4        44324735                               [50]               [2]   \n",
       "...           ...                                ...               ...   \n",
       "7680      4559574                               [20]         [57,  19]   \n",
       "7681     42535671                               [20]         [57,  33]   \n",
       "7682      2446129                               [25]          [6,  83]   \n",
       "7683      9515318                               [23]          [6,  84]   \n",
       "7684      9407346                               [50]         [71,  15]   \n",
       "\n",
       "     Solvent (Reaction Details)  \\\n",
       "0             [tetrahydrofuran]   \n",
       "1      [N,N-dimethyl-formamide]   \n",
       "2             [tetrahydrofuran]   \n",
       "3             [tetrahydrofuran]   \n",
       "4             [tetrahydrofuran]   \n",
       "...                         ...   \n",
       "7680          [tetrahydrofuran]   \n",
       "7681                  [toluene]   \n",
       "7682   [N,N-dimethyl-formamide]   \n",
       "7683                  [toluene]   \n",
       "7684          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     [CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...   \n",
       "1     [N#C[Zn]C#N, CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(...   \n",
       "2      [COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]   \n",
       "3     [Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...   \n",
       "4     [BrC1=CC=C2NC=C(C(=O)[C@@H]3CCCN3C(=O)OCC3=CC=...   \n",
       "...                                                 ...   \n",
       "7680      [CC1=CN(=O)=C(C)C(Cl)=N1, CC(C)CC[Zn]CCC(C)C]   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(Br)C(Br)=C(Br)N3[B](F)(F...   \n",
       "7682  [FC(F)(F)S(=O)(=O)OC1=CCCCC1=O, CCCC\\C=C\\[Zn]\\...   \n",
       "7683               [Br[Zn]C1=NC=CS1, CCCCCC\\C=C(/Br)Br]   \n",
       "7684   [C[Zn]Br, ClC1=NC2=C(N=CN2CC2=CC=CC=C2)C(Cl)=N1]   \n",
       "\n",
       "                                               products  \\\n",
       "0     [CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...   \n",
       "1      [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]   \n",
       "2                [COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]   \n",
       "3     [CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...   \n",
       "4     [O=C(OCC1=CC=CC=C1)N1CCC[C@H]1C(=O)C1=CNC2=C1C...   \n",
       "...                                                 ...   \n",
       "7680  [CC(C)CCC1=C(C)N=CC(C)=N1, CC(C)CCC1=NC(C)=CN(...   \n",
       "7681  [CCC1=C(C)C2=C(C)C3=C(C)C(C)=C(C)N3[B](F)(F)[N...   \n",
       "7682          [CCCC\\C=C\\C1=CCCCC1=O, CCCC\\C=C\\C=C\\CCCC]   \n",
       "7683     [CCCCCCC#CC1=NC=CS1, CCCCCC\\C=C(Br)\\C1=NC=CS1]   \n",
       "7684  [CC1=NC(Cl)=NC2=C1N=CN2CC1=CC=CC=C1, CC1=NC2=C...   \n",
       "\n",
       "                                               reagents  \n",
       "0                         [palladium diacetate, ruphos]  \n",
       "1     [zinc, tris-(dibenzylideneacetone)dipalladium(0)]  \n",
       "2           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "4     [((2-dicyclohexylphosphino-2',4?,6?-triisoprop...  \n",
       "...                                                 ...  \n",
       "7680  [1,3-bis[(diphenylphosphino)propane]dichloroni...  \n",
       "7681    [bis-triphenylphosphine-palladium(II) chloride]  \n",
       "7682  [bis-triphenylphosphine-palladium(II) chloride...  \n",
       "7683                                   [Cl2Pd(DPEphos)]  \n",
       "7684        [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "\n",
       "[7406 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove duplicated reagents, reactants, solvents, products, temperature\n",
    "# duplicate compounds and reagents often just refer to multiple equivalents\n",
    "# duplicate solvents may refer to materials being added together as solutions\n",
    "str_cols = ['reagents', 'reactants', 'products', 'Solvent (Reaction Details)']\n",
    "            \n",
    "int_cols = ['Temperature (Reaction Details) [C]']\n",
    "for col in str_cols:\n",
    "    negishi_df[col] = negishi_df[col].apply(lambda x: list(set(map(str.strip, x))))\n",
    "\n",
    "for col in int_cols:\n",
    "    #negishi_df[col] = negishi_df[col].apply(lambda x: list(set(x)))\n",
    "    negishi_df[col] = negishi_df[col].apply(lambda x: list(set(filter(None, re.split('; |, | - |[a-zA-Z]+|([0-9]+)-', str(x))))))\n",
    "\n",
    "\n",
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean reactions outside of desired structural format\n",
    "Some reactions may have excessive numbers of reactants, products, reagents, etc. listed. We remove those reactions to maintain a consistent format for our prediction task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6478 entries, 0 to 7669\n",
      "Data columns (total 6 columns):\n",
      " #   Column                              Non-Null Count  Dtype \n",
      "---  ------                              --------------  ----- \n",
      " 0   Reaction ID                         6478 non-null   int64 \n",
      " 1   Temperature (Reaction Details) [C]  6478 non-null   object\n",
      " 2   Solvent (Reaction Details)          6478 non-null   object\n",
      " 3   reactants                           6478 non-null   object\n",
      " 4   products                            6478 non-null   object\n",
      " 5   reagents                            6478 non-null   object\n",
      "dtypes: int64(1), object(5)\n",
      "memory usage: 354.3+ KB\n"
     ]
    }
   ],
   "source": [
    "# remove reactions with > 2 reactants, > 1 product, > 3 solvents,\n",
    "# > 4 reagents, and/or > 1 yield\n",
    "negishi_df = negishi_df[negishi_df['reactants'].map(len) < 3]\n",
    "negishi_df = negishi_df[negishi_df['products'].map(len) < 2]\n",
    "negishi_df = negishi_df[negishi_df['Solvent (Reaction Details)'].map(len) < 4]\n",
    "negishi_df = negishi_df[negishi_df['reagents'].map(len) < 5]\n",
    "negishi_df = negishi_df[negishi_df['Yield (numerical)'].map(len) < 2]\n",
    "\n",
    "# drop yield column (optional)\n",
    "negishi_df = negishi_df.drop(['Yield (numerical)'], axis=1)\n",
    "\n",
    "# get overview of missing values and dtypes \n",
    "negishi_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reaction ID</th>\n",
       "      <th>Temperature (Reaction Details) [C]</th>\n",
       "      <th>Solvent (Reaction Details)</th>\n",
       "      <th>reactants</th>\n",
       "      <th>products</th>\n",
       "      <th>reagents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36844980</td>\n",
       "      <td>[60]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...</td>\n",
       "      <td>[CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...</td>\n",
       "      <td>[palladium diacetate, ruphos]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28516312</td>\n",
       "      <td>[100]</td>\n",
       "      <td>[N,N-dimethyl-formamide]</td>\n",
       "      <td>[N#C[Zn]C#N, CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(...</td>\n",
       "      <td>[CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]</td>\n",
       "      <td>[zinc, tris-(dibenzylideneacetone)dipalladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24802995</td>\n",
       "      <td>[100]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]</td>\n",
       "      <td>[COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41263899</td>\n",
       "      <td>[65]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...</td>\n",
       "      <td>[CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1807174</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[BrC1=CC2=C(NC1=O)C=CC=C2, Cl[Zn]C1=CN=CC=C1]</td>\n",
       "      <td>[O=C1NC2=CC=CC=C2C=C1C1=CN=CC=C1]</td>\n",
       "      <td>[tetrakis(triphenylphosphine) palladium(0)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7664</th>\n",
       "      <td>9520363</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[1,4-dioxane]</td>\n",
       "      <td>[CC(=O)C1=C(Br)C=CC=C1, C[Zn]C]</td>\n",
       "      <td>[CC(=O)C1=CC=CC=C1C]</td>\n",
       "      <td>[(1,2-bis(diphenylphosphino)ethane)palladium(I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7665</th>\n",
       "      <td>9521618</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[1,4-dioxane]</td>\n",
       "      <td>[OCCC1=CC=C(Br)C=C1, C[Zn]C]</td>\n",
       "      <td>[CC1=CC=C(CCO)C=C1]</td>\n",
       "      <td>[(1,1'-bis(diphenylphosphino)ferrocene)palladi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7666</th>\n",
       "      <td>9523747</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[1,4-dioxane]</td>\n",
       "      <td>[OC(=O)C1=C(C=C(Br)C=C1)N(=O)=O, C[Zn]C]</td>\n",
       "      <td>[CC1=CC=C(C(O)=O)C(=C1)N(=O)=O]</td>\n",
       "      <td>[(1,1'-bis(diphenylphosphino)ferrocene)palladi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7667</th>\n",
       "      <td>9529510</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[1,4-dioxane]</td>\n",
       "      <td>[CC(=O)C1=CC=C(Br)C=C1, C[Zn]C]</td>\n",
       "      <td>[CC(=O)C1=CC=C(C)C=C1]</td>\n",
       "      <td>[(1,1'-bis(diphenylphosphino)ferrocene)palladi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7669</th>\n",
       "      <td>29846412</td>\n",
       "      <td>[20]</td>\n",
       "      <td>[tetrahydrofuran]</td>\n",
       "      <td>[COC(=O)C1=C(I)C=CC(O[Si](C(C)C)(C(C)C)C(C)C)=...</td>\n",
       "      <td>[COC(=O)C1=C(C=C)C=CC(O[Si](C(C)C)(C(C)C)C(C)C...</td>\n",
       "      <td>[bis-triphenylphosphine-palladium(II) chloride]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6478 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Reaction ID Temperature (Reaction Details) [C]  \\\n",
       "0        36844980                               [60]   \n",
       "1        28516312                              [100]   \n",
       "2        24802995                              [100]   \n",
       "3        41263899                               [65]   \n",
       "6         1807174                               [20]   \n",
       "...           ...                                ...   \n",
       "7664      9520363                               [20]   \n",
       "7665      9521618                               [20]   \n",
       "7666      9523747                               [20]   \n",
       "7667      9529510                               [20]   \n",
       "7669     29846412                               [20]   \n",
       "\n",
       "     Solvent (Reaction Details)  \\\n",
       "0             [tetrahydrofuran]   \n",
       "1      [N,N-dimethyl-formamide]   \n",
       "2             [tetrahydrofuran]   \n",
       "3             [tetrahydrofuran]   \n",
       "6             [tetrahydrofuran]   \n",
       "...                         ...   \n",
       "7664              [1,4-dioxane]   \n",
       "7665              [1,4-dioxane]   \n",
       "7666              [1,4-dioxane]   \n",
       "7667              [1,4-dioxane]   \n",
       "7669          [tetrahydrofuran]   \n",
       "\n",
       "                                              reactants  \\\n",
       "0     [CC(C)(C)OC(=O)NC1(CCC2=CC=C(Cl)C=C2)COC(C)(C)...   \n",
       "1     [N#C[Zn]C#N, CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(...   \n",
       "2      [COC1=C(OC)C=C2C(Br)=CN=NC2=C1, Br[Zn]C1=NC=CS1]   \n",
       "3     [Br[Zn]C1=CC=CN=C1, CSC1=NC(=NC2=NC(N)=C(C#N)C...   \n",
       "6         [BrC1=CC2=C(NC1=O)C=CC=C2, Cl[Zn]C1=CN=CC=C1]   \n",
       "...                                                 ...   \n",
       "7664                    [CC(=O)C1=C(Br)C=CC=C1, C[Zn]C]   \n",
       "7665                       [OCCC1=CC=C(Br)C=C1, C[Zn]C]   \n",
       "7666           [OC(=O)C1=C(C=C(Br)C=C1)N(=O)=O, C[Zn]C]   \n",
       "7667                    [CC(=O)C1=CC=C(Br)C=C1, C[Zn]C]   \n",
       "7669  [COC(=O)C1=C(I)C=CC(O[Si](C(C)C)(C(C)C)C(C)C)=...   \n",
       "\n",
       "                                               products  \\\n",
       "0     [CCCCCCCCC1=CC=C(CCC2(COC(C)(C)OC2)NC(=O)OC(C)...   \n",
       "1      [CC(C)C1=C(NC(=O)NC1=O)C(=O)C1=CC(C)=CC(=C1)C#N]   \n",
       "2                [COC1=C(OC)C=C2C(=C1)N=NC=C2C1=NC=CS1]   \n",
       "3     [CC(C)(C)C1=NC2=NC(N)=C(C#N)C(N)=C2C(=N1)C1=CC...   \n",
       "6                     [O=C1NC2=CC=CC=C2C=C1C1=CN=CC=C1]   \n",
       "...                                                 ...   \n",
       "7664                               [CC(=O)C1=CC=CC=C1C]   \n",
       "7665                                [CC1=CC=C(CCO)C=C1]   \n",
       "7666                    [CC1=CC=C(C(O)=O)C(=C1)N(=O)=O]   \n",
       "7667                             [CC(=O)C1=CC=C(C)C=C1]   \n",
       "7669  [COC(=O)C1=C(C=C)C=CC(O[Si](C(C)C)(C(C)C)C(C)C...   \n",
       "\n",
       "                                               reagents  \n",
       "0                         [palladium diacetate, ruphos]  \n",
       "1     [zinc, tris-(dibenzylideneacetone)dipalladium(0)]  \n",
       "2           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "3           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "6           [tetrakis(triphenylphosphine) palladium(0)]  \n",
       "...                                                 ...  \n",
       "7664  [(1,2-bis(diphenylphosphino)ethane)palladium(I...  \n",
       "7665  [(1,1'-bis(diphenylphosphino)ferrocene)palladi...  \n",
       "7666  [(1,1'-bis(diphenylphosphino)ferrocene)palladi...  \n",
       "7667  [(1,1'-bis(diphenylphosphino)ferrocene)palladi...  \n",
       "7669    [bis-triphenylphosphine-palladium(II) chloride]  \n",
       "\n",
       "[6478 rows x 6 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualize change\n",
    "negishi_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dictionary construction\n",
    "A \"dictionary\" of statistically relevant reaction conditions is constructed to define the downstream prediction space. This process includes analyzing the distribution of reported instances for all unique reagents found above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reagent</th>\n",
       "      <th>count</th>\n",
       "      <th>frequency</th>\n",
       "      <th>cumulative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>1898</td>\n",
       "      <td>0.203561</td>\n",
       "      <td>0.203561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>449</td>\n",
       "      <td>0.048155</td>\n",
       "      <td>0.251716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>415</td>\n",
       "      <td>0.044509</td>\n",
       "      <td>0.296225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>palladium diacetate</td>\n",
       "      <td>356</td>\n",
       "      <td>0.038181</td>\n",
       "      <td>0.334406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(1,1'-bis(diphenylphosphino)ferrocene)palladiu...</td>\n",
       "      <td>332</td>\n",
       "      <td>0.035607</td>\n",
       "      <td>0.370013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>2-iodo-propane</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000107</td>\n",
       "      <td>0.999571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>[1,3- bisdiphenylphosphino]Ni(II)Cl2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000107</td>\n",
       "      <td>0.999678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>1-azido-1lambda3-benzo[d][1,2]iodaoxol-3(1H)-one</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000107</td>\n",
       "      <td>0.999785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>potassium methanolate</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000107</td>\n",
       "      <td>0.999893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>(Z)-1,2-dibromo-ethene</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000107</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>364 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               reagent  count  frequency  \\\n",
       "0            tetrakis(triphenylphosphine) palladium(0)   1898   0.203561   \n",
       "1            tris-(dibenzylideneacetone)dipalladium(0)    449   0.048155   \n",
       "2        bis-triphenylphosphine-palladium(II) chloride    415   0.044509   \n",
       "3                                  palladium diacetate    356   0.038181   \n",
       "4    (1,1'-bis(diphenylphosphino)ferrocene)palladiu...    332   0.035607   \n",
       "..                                                 ...    ...        ...   \n",
       "359                                     2-iodo-propane      1   0.000107   \n",
       "360               [1,3- bisdiphenylphosphino]Ni(II)Cl2      1   0.000107   \n",
       "361   1-azido-1lambda3-benzo[d][1,2]iodaoxol-3(1H)-one      1   0.000107   \n",
       "362                              potassium methanolate      1   0.000107   \n",
       "363                             (Z)-1,2-dibromo-ethene      1   0.000107   \n",
       "\n",
       "     cumulative  \n",
       "0      0.203561  \n",
       "1      0.251716  \n",
       "2      0.296225  \n",
       "3      0.334406  \n",
       "4      0.370013  \n",
       "..          ...  \n",
       "359    0.999571  \n",
       "360    0.999678  \n",
       "361    0.999785  \n",
       "362    0.999893  \n",
       "363    1.000000  \n",
       "\n",
       "[364 rows x 4 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dictionaries for label space \n",
    "# return list of all unique reagent names and frequencies to be sorted\n",
    "# into categories metal, ligand, base, and additive\n",
    "reagent_all = negishi_df['reagents'].explode()\n",
    "\n",
    "# combine into single list\n",
    "reagent_list = list(zip(reagent_all.value_counts().index,\n",
    "                        reagent_all.value_counts(),\n",
    "                        reagent_all.value_counts(normalize=True)\n",
    "                       )\n",
    "                   )\n",
    "\n",
    "# create reagent dataframe for processing\n",
    "reagent_columns = ['reagent', 'count', 'frequency']\n",
    "reagent_df = pd.DataFrame(reagent_list, columns=reagent_columns)\n",
    "\n",
    "# add column for cumulative sum of frequency\n",
    "reagent_df['cumulative'] = reagent_df.frequency.cumsum()\n",
    "\n",
    "# visualize dataframe\n",
    "reagent_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove infrequent reagents to avoid issues of sparsity\n",
    "This may include sparsely-reported reagents/conditions, or may even include cases of rare or mis-spellings of a common reagent. While addressing all of these cases is potentially manageable in this smaller dataset, this is not the case with larger datasets with several thousand unique reagents. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reagent</th>\n",
       "      <th>count</th>\n",
       "      <th>frequency</th>\n",
       "      <th>cumulative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>1898</td>\n",
       "      <td>0.203561</td>\n",
       "      <td>0.203561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>449</td>\n",
       "      <td>0.048155</td>\n",
       "      <td>0.251716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>415</td>\n",
       "      <td>0.044509</td>\n",
       "      <td>0.296225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>palladium diacetate</td>\n",
       "      <td>356</td>\n",
       "      <td>0.038181</td>\n",
       "      <td>0.334406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(1,1'-bis(diphenylphosphino)ferrocene)palladiu...</td>\n",
       "      <td>332</td>\n",
       "      <td>0.035607</td>\n",
       "      <td>0.370013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Pt2Me4(SMe2)2</td>\n",
       "      <td>8</td>\n",
       "      <td>0.000858</td>\n",
       "      <td>0.946268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>[1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...</td>\n",
       "      <td>7</td>\n",
       "      <td>0.000751</td>\n",
       "      <td>0.947018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>SPhos Pd G4</td>\n",
       "      <td>7</td>\n",
       "      <td>0.000751</td>\n",
       "      <td>0.947769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>triethylamine</td>\n",
       "      <td>7</td>\n",
       "      <td>0.000751</td>\n",
       "      <td>0.948520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>copper(l) chloride</td>\n",
       "      <td>7</td>\n",
       "      <td>0.000751</td>\n",
       "      <td>0.949271</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>129 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               reagent  count  frequency  \\\n",
       "0            tetrakis(triphenylphosphine) palladium(0)   1898   0.203561   \n",
       "1            tris-(dibenzylideneacetone)dipalladium(0)    449   0.048155   \n",
       "2        bis-triphenylphosphine-palladium(II) chloride    415   0.044509   \n",
       "3                                  palladium diacetate    356   0.038181   \n",
       "4    (1,1'-bis(diphenylphosphino)ferrocene)palladiu...    332   0.035607   \n",
       "..                                                 ...    ...        ...   \n",
       "124                                      Pt2Me4(SMe2)2      8   0.000858   \n",
       "125  [1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...      7   0.000751   \n",
       "126                                        SPhos Pd G4      7   0.000751   \n",
       "127                                      triethylamine      7   0.000751   \n",
       "128                                 copper(l) chloride      7   0.000751   \n",
       "\n",
       "     cumulative  \n",
       "0      0.203561  \n",
       "1      0.251716  \n",
       "2      0.296225  \n",
       "3      0.334406  \n",
       "4      0.370013  \n",
       "..          ...  \n",
       "124    0.946268  \n",
       "125    0.947018  \n",
       "126    0.947769  \n",
       "127    0.948520  \n",
       "128    0.949271  \n",
       "\n",
       "[129 rows x 4 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter to reagents within top 95% by frequency\n",
    "reagent_df = reagent_df[reagent_df['cumulative'] <= 0.95]\n",
    "\n",
    "# visualize change\n",
    "reagent_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export current dataframe for category labeling\n",
    "Each reagent is given a label based on its role in the reaction. For example, reagents in this dataset were categorized as either metal sources (m), ligands (l), or additives (a). The appropriate categories will depend on the nature of the reaction being studied and requires some background understanding of the reaction's typical conditions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save reagent dataframe to csv for category (i.e. reaction role) sorting\n",
    "reagent_df.to_csv('negishi_reagent_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>reagent</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>m</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>1898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>m</td>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>m</td>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>m</td>\n",
       "      <td>palladium diacetate</td>\n",
       "      <td>356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>m</td>\n",
       "      <td>(1,1'-bis(diphenylphosphino)ferrocene)palladiu...</td>\n",
       "      <td>332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>m</td>\n",
       "      <td>tetrakis(triphenylphosphine)nickel(0)</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>m</td>\n",
       "      <td>SPhos Pd G4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>m</td>\n",
       "      <td>[1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>a</td>\n",
       "      <td>sodium hydroxide</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>a</td>\n",
       "      <td>triethylamine</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>129 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    category                                            reagent  count\n",
       "0          m          tetrakis(triphenylphosphine) palladium(0)   1898\n",
       "1          m          tris-(dibenzylideneacetone)dipalladium(0)    449\n",
       "2          m      bis-triphenylphosphine-palladium(II) chloride    415\n",
       "3          m                                palladium diacetate    356\n",
       "4          m  (1,1'-bis(diphenylphosphino)ferrocene)palladiu...    332\n",
       "..       ...                                                ...    ...\n",
       "124        m              tetrakis(triphenylphosphine)nickel(0)      8\n",
       "125        m                                        SPhos Pd G4      7\n",
       "126        m  [1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...      7\n",
       "127        a                                   sodium hydroxide      7\n",
       "128        a                                      triethylamine      7\n",
       "\n",
       "[129 rows x 3 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reload labeled dataframe (negishi_reagent_df condensed to nr_df)\n",
    "# new column 'category' has a shorthand label for the type of reagent in\n",
    "# metal (m), ligand (l), additive (a). only importing 'category', 'reagent',\n",
    "# and 'count' columns since will be recounting per category frequencies\n",
    "nr_df_labeled = pd.read_csv('negishi_reagent_df.csv',\n",
    "                            usecols=['category', 'reagent', 'count'])\n",
    "\n",
    "# visualize dataframe\n",
    "nr_df_labeled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract the categorized reagents "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>reagent</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>lithium chloride</td>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>l</td>\n",
       "      <td>1,1'-bis-(diphenylphosphino)ferrocene</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>m</td>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>1898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category                                    reagent  count\n",
       "0        a                           lithium chloride    242\n",
       "1        l      1,1'-bis-(diphenylphosphino)ferrocene    198\n",
       "2        m  tetrakis(triphenylphosphine) palladium(0)   1898"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# group dataframe by category labels (note: df.groupby preserves order)\n",
    "nrl_grouped = nr_df_labeled.groupby('category', as_index=False)\n",
    "\n",
    "# see the first label in each group\n",
    "nrl_grouped.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct solvent dictionary\n",
    "Applies same processing as above to solvent field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>count</th>\n",
       "      <th>frequency</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tetrahydrofuran</td>\n",
       "      <td>4599</td>\n",
       "      <td>0.545099</td>\n",
       "      <td>0.545099</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N,N-dimethyl-formamide</td>\n",
       "      <td>900</td>\n",
       "      <td>0.106673</td>\n",
       "      <td>0.651772</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1-methyl-pyrrolidin-2-one</td>\n",
       "      <td>681</td>\n",
       "      <td>0.080716</td>\n",
       "      <td>0.732488</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>toluene</td>\n",
       "      <td>545</td>\n",
       "      <td>0.064596</td>\n",
       "      <td>0.797084</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1,4-dioxane</td>\n",
       "      <td>335</td>\n",
       "      <td>0.039706</td>\n",
       "      <td>0.836790</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>N,N-dimethyl acetamide</td>\n",
       "      <td>248</td>\n",
       "      <td>0.029394</td>\n",
       "      <td>0.866185</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>hexane</td>\n",
       "      <td>214</td>\n",
       "      <td>0.025364</td>\n",
       "      <td>0.891549</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>diethyl ether</td>\n",
       "      <td>204</td>\n",
       "      <td>0.024179</td>\n",
       "      <td>0.915728</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>water</td>\n",
       "      <td>122</td>\n",
       "      <td>0.014460</td>\n",
       "      <td>0.930188</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DMF (N,N-dimethyl-formamide)</td>\n",
       "      <td>102</td>\n",
       "      <td>0.012090</td>\n",
       "      <td>0.942278</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          label  count  frequency  cumulative category\n",
       "0               tetrahydrofuran   4599   0.545099    0.545099        s\n",
       "1        N,N-dimethyl-formamide    900   0.106673    0.651772        s\n",
       "2     1-methyl-pyrrolidin-2-one    681   0.080716    0.732488        s\n",
       "3                       toluene    545   0.064596    0.797084        s\n",
       "4                   1,4-dioxane    335   0.039706    0.836790        s\n",
       "5        N,N-dimethyl acetamide    248   0.029394    0.866185        s\n",
       "6                        hexane    214   0.025364    0.891549        s\n",
       "7                 diethyl ether    204   0.024179    0.915728        s\n",
       "8                         water    122   0.014460    0.930188        s\n",
       "9  DMF (N,N-dimethyl-formamide)    102   0.012090    0.942278        s"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# return list of all unique solvent names and frequencies\n",
    "solvent_all = negishi_df['Solvent (Reaction Details)'].explode()\n",
    "\n",
    "# combine into single list with counts and frequencies\n",
    "solvent_list = list(zip(solvent_all.value_counts().index,\n",
    "                        solvent_all.value_counts(),\n",
    "                        solvent_all.value_counts(normalize=True)\n",
    "                       )\n",
    "                   )\n",
    "\n",
    "# create solvent dataframe for processing\n",
    "solvent_columns = ['label', 'count', 'frequency']\n",
    "solvent_df = pd.DataFrame(solvent_list, columns=solvent_columns)\n",
    "\n",
    "# add column for cumulative sum of frequency and category label 's'\n",
    "solvent_df['cumulative'] = solvent_df.frequency.cumsum()\n",
    "solvent_df['category'] = 's'\n",
    "\n",
    "# filter to solvents within top 95% by frequency\n",
    "solvent_df = solvent_df[solvent_df['cumulative'] <= 0.95]\n",
    "\n",
    "# visualize dataframe\n",
    "solvent_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define temperature quantiles based on distribution in dataset\n",
    "This process can be applied to any continuous reaction variable such as temperature, pressure, concentration, as well as analysis metrics such as yield and selectivity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        60.0\n",
       "1       100.0\n",
       "2       100.0\n",
       "3        65.0\n",
       "6        20.0\n",
       "        ...  \n",
       "7664     20.0\n",
       "7665     20.0\n",
       "7666     20.0\n",
       "7667     20.0\n",
       "7669     20.0\n",
       "Name: Temperature (Reaction Details) [C], Length: 7280, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split temperatures by whitespace and delimeters\n",
    "temp = 'Temperature (Reaction Details) [C]'\n",
    "\n",
    "# create series of all temperatures\n",
    "temperature_all = negishi_df[temp].explode()\n",
    "\n",
    "# convert series to float (took a lot of trial and error with regex of list\n",
    "# splitting above to get only integers)\n",
    "temperature_all = temperature_all.astype(float)\n",
    "\n",
    "# check conversion (took a lot of )\n",
    "temperature_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min: -163.0\n",
      "max: 44124.0\n"
     ]
    }
   ],
   "source": [
    "# check minimum and maximum temperatures (these may be outliers)\n",
    "print('min:', temperature_all.min())\n",
    "print('max:', temperature_all.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x11897d1d0>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATtklEQVR4nO3db4xd9Z3f8fdnDWFXm2yBxSDXdmsn9bYhldbQKVClXaVJFgxRZaIWiTxYrAjJWwmkRN1WNZsHZJMikaoJKmqWyiluzCobFm0SYSVsWS+bNIpU/pisYzBelgnQMLGFZ2tCEqWlhf32wf25vZg7M3fGM3cMv/dLurrnfs/v3PM9x3c+c+bcc69TVUiS+vBzq92AJGlyDH1J6oihL0kdMfQlqSOGviR15KzVbmA+F1xwQW3atGm125CkN5XHH3/8L6tq7ah5Z3Tob9q0iQMHDqx2G5L0ppLkv881z9M7ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkTP6E7kSwKZd31jtFkZ6/vYPrXYL0qJ5pC9JHVkw9JP8fJJHk3wvyeEkv9PqX0zyXJKD7ba11ZPkziTTSQ4luXTouXYkeabddqzcZkmSRhnn9M4rwPur6qdJzga+k+SP2rx/VVV/eMr4q4Et7XY5cBdweZLzgVuBKaCAx5Psq6qXlmNDJEkLW/BIvwZ+2h6e3W7z/W/q24F72nIPA+cmWQdcBeyvqhMt6PcD206vfUnSYox1Tj/JmiQHgeMMgvuRNuu2dgrnjiTntNp64IWhxWdaba66JGlCxgr9qnqtqrYCG4DLkvxd4Bbg7wB/Hzgf+NdteEY9xTz110myM8mBJAdmZ2fHaU+SNKZFXb1TVT8CvgVsq6pj7RTOK8B/Bi5rw2aAjUOLbQCOzlM/dR27q2qqqqbWrh35H79IkpZonKt31iY5t03/AvBB4M/beXqSBLgWeLItsg+4oV3FcwXwclUdAx4ErkxyXpLzgCtbTZI0IeNcvbMO2JtkDYNfEvdV1deT/GmStQxO2xwE/nkb/wBwDTAN/Az4KEBVnUjyaeCxNu5TVXVi+TZFkrSQBUO/qg4Bl4yov3+O8QXcNMe8PcCeRfYoSVomfiJXkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWTD0k/x8kkeTfC/J4SS/0+qbkzyS5Jkkf5Dkba1+Tns83eZvGnquW1r96SRXrdRGSZJGG+dI/xXg/VX1q8BWYFuSK4DPAHdU1RbgJeDGNv5G4KWq+lvAHW0cSS4GrgfeA2wDfjfJmuXcGEnS/BYM/Rr4aXt4drsV8H7gD1t9L3Btm97eHtPmfyBJWv3eqnqlqp4DpoHLlmUrJEljGeucfpI1SQ4Cx4H9wPeBH1XVq23IDLC+Ta8HXgBo818Gfnm4PmKZ4XXtTHIgyYHZ2dnFb5EkaU5jhX5VvVZVW4ENDI7O3z1qWLvPHPPmqp+6rt1VNVVVU2vXrh2nPUnSmBZ19U5V/Qj4FnAFcG6Ss9qsDcDRNj0DbARo8/8acGK4PmIZSdIEjHP1ztok57bpXwA+CBwBvgn8szZsB3B/m97XHtPm/2lVVatf367u2QxsAR5drg2RJC3srIWHsA7Y2660+Tngvqr6epKngHuT/Bvgz4C72/i7gd9LMs3gCP96gKo6nOQ+4CngVeCmqnpteTdHkjSfBUO/qg4Bl4yoP8uIq2+q6n8B183xXLcBty2+TUnScvATuZLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siCoZ9kY5JvJjmS5HCSj7X6J5P8MMnBdrtmaJlbkkwneTrJVUP1ba02nWTXymySJGkuZ40x5lXgt6rqu0neATyeZH+bd0dV/bvhwUkuBq4H3gP8deBPkvxKm/154NeBGeCxJPuq6qnl2BBJ0sIWDP2qOgYca9M/SXIEWD/PItuBe6vqFeC5JNPAZW3edFU9C5Dk3jbW0JekCVnUOf0km4BLgEda6eYkh5LsSXJeq60HXhhabKbV5qqfuo6dSQ4kOTA7O7uY9iRJCxg79JO8HfgK8PGq+jFwF/AuYCuDvwQ+e3LoiMVrnvrrC1W7q2qqqqbWrl07bnuSpDGMc06fJGczCPwvVdVXAarqxaH5XwC+3h7OABuHFt8AHG3Tc9UlSRMwztU7Ae4GjlTV54bq64aGfRh4sk3vA65Pck6SzcAW4FHgMWBLks1J3sbgzd59y7MZkqRxjHOk/17gN4Ankhxstd8GPpJkK4NTNM8DvwlQVYeT3MfgDdpXgZuq6jWAJDcDDwJrgD1VdXgZt0WStIBxrt75DqPPxz8wzzK3AbeNqD8w33KSpJU11jl9SW+0adc3VruFkZ6//UOr3YLOYH4NgyR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRBUM/ycYk30xyJMnhJB9r9fOT7E/yTLs/r9WT5M4k00kOJbl06Ll2tPHPJNmxcpslSRplnCP9V4Hfqqp3A1cANyW5GNgFPFRVW4CH2mOAq4Et7bYTuAsGvySAW4HLgcuAW0/+opAkTcaCoV9Vx6rqu236J8ARYD2wHdjbhu0Frm3T24F7auBh4Nwk64CrgP1VdaKqXgL2A9uWdWskSfNa1Dn9JJuAS4BHgIuq6hgMfjEAF7Zh64EXhhababW56qeuY2eSA0kOzM7OLqY9SdICxg79JG8HvgJ8vKp+PN/QEbWap/76QtXuqpqqqqm1a9eO254kaQxjhX6SsxkE/peq6qut/GI7bUO7P97qM8DGocU3AEfnqUuSJmScq3cC3A0cqarPDc3aB5y8AmcHcP9Q/YZ2Fc8VwMvt9M+DwJVJzmtv4F7ZapKkCTlrjDHvBX4DeCLJwVb7beB24L4kNwI/AK5r8x4ArgGmgZ8BHwWoqhNJPg081sZ9qqpOLMtWSJLGsmDoV9V3GH0+HuADI8YXcNMcz7UH2LOYBiVJy8dP5EpSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWTD0k+xJcjzJk0O1Tyb5YZKD7XbN0LxbkkwneTrJVUP1ba02nWTX8m+KJGkh4xzpfxHYNqJ+R1VtbbcHAJJcDFwPvKct87tJ1iRZA3weuBq4GPhIGytJmqCzFhpQVd9OsmnM59sO3FtVrwDPJZkGLmvzpqvqWYAk97axTy26Y0nSkp3OOf2bkxxqp3/Oa7X1wAtDY2Zaba66JGmClhr6dwHvArYCx4DPtnpGjK156m+QZGeSA0kOzM7OLrE9SdIoSwr9qnqxql6rqr8CvsD/P4UzA2wcGroBODpPfdRz766qqaqaWrt27VLakyTNYUmhn2Td0MMPAyev7NkHXJ/knCSbgS3Ao8BjwJYkm5O8jcGbvfuW3rYkaSkWfCM3yZeB9wEXJJkBbgXel2Qrg1M0zwO/CVBVh5Pcx+AN2leBm6rqtfY8NwMPAmuAPVV1eNm3RpI0r3Gu3vnIiPLd84y/DbhtRP0B4IFFdSdJWlZ+IleSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR1ZMPST7ElyPMmTQ7Xzk+xP8ky7P6/Vk+TOJNNJDiW5dGiZHW38M0l2rMzmSJLmM86R/heBbafUdgEPVdUW4KH2GOBqYEu77QTugsEvCeBW4HLgMuDWk78oJEmTs2DoV9W3gROnlLcDe9v0XuDaofo9NfAwcG6SdcBVwP6qOlFVLwH7eeMvEknSClvqOf2LquoYQLu/sNXXAy8MjZtptbnqb5BkZ5IDSQ7Mzs4usT1J0ihnLfPzZUSt5qm/sVi1G9gNMDU1NXKMpLlt2vWN1W5hpOdv/9BqtyCWfqT/YjttQ7s/3uozwMahcRuAo/PUJUkTtNTQ3wecvAJnB3D/UP2GdhXPFcDL7fTPg8CVSc5rb+Be2WqSpAla8PROki8D7wMuSDLD4Cqc24H7ktwI/AC4rg1/ALgGmAZ+BnwUoKpOJPk08Fgb96mqOvXNYUnSClsw9KvqI3PM+sCIsQXcNMfz7AH2LKo7SdKy8hO5ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqyGmFfpLnkzyR5GCSA612fpL9SZ5p9+e1epLcmWQ6yaEkly7HBkiSxrccR/r/uKq2VtVUe7wLeKiqtgAPtccAVwNb2m0ncNcyrFuStAgrcXpnO7C3Te8Frh2q31MDDwPnJlm3AuuXJM3hdEO/gD9O8niSna12UVUdA2j3F7b6euCFoWVnWu11kuxMciDJgdnZ2dNsT5I07KzTXP69VXU0yYXA/iR/Ps/YjKjVGwpVu4HdAFNTU2+YL0lautM60q+qo+3+OPA14DLgxZOnbdr98TZ8Btg4tPgG4OjprF+StDhLDv0kv5jkHSengSuBJ4F9wI42bAdwf5veB9zQruK5Anj55GkgSdJknM7pnYuAryU5+Ty/X1X/JcljwH1JbgR+AFzXxj8AXANMAz8DPnoa65YkLcGSQ7+qngV+dUT9fwAfGFEv4Kalrk+SdPr8RK4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOnK6/4mKJI1l065vrHYLIz1/+4dWu4WJ8khfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOTPw6/STbgH8PrAH+U1XdPukeNNqZeh21pOUz0dBPsgb4PPDrwAzwWJJ9VfXUJPtYbYarpNUy6SP9y4DpqnoWIMm9wHagq9CXdOY4Uw/CVuqTwpMO/fXAC0OPZ4DLhwck2QnsbA9/muTp01jfBcBfnsbyK8W+Fse+Fse+FueM7CufOa2+/uZcMyYd+hlRq9c9qNoN7F6WlSUHqmpqOZ5rOdnX4tjX4tjX4vTW16Sv3pkBNg493gAcnXAPktStSYf+Y8CWJJuTvA24Htg34R4kqVsTPb1TVa8muRl4kMElm3uq6vAKrnJZThOtAPtaHPtaHPtanK76SlUtPEqS9JbgJ3IlqSOGviR15C0R+kmuS3I4yV8lmRqqb0ryP5McbLf/ODTv7yV5Isl0kjuTjLqcdEX6avNuaet+OslVQ/VtrTadZNdy9zSix08m+eHQPrpmoR4nZdL7YoFenm+vl4NJDrTa+Un2J3mm3Z83oV72JDme5Mmh2sheMnBn24eHklw64b5W9fWVZGOSbyY50n4WP9bqq7q/5ulr5fdXVb3pb8C7gb8NfAuYGqpvAp6cY5lHgX/A4LMDfwRcPcG+Lga+B5wDbAa+z+CN7TVt+p3A29qYi1d4330S+Jcj6iN7nOC/6cT3xQL9PA9ccErt3wK72vQu4DMT6uXXgEuHX9tz9QJc017fAa4AHplwX6v6+gLWAZe26XcAf9HWvar7a56+Vnx/vSWO9KvqSFWN/cndJOuAX6qq/1aDPXoPcO0E+9oO3FtVr1TVc8A0g6+o+H9fU1FV/xs4+TUVq2GuHiflTNoXc9kO7G3Te1mB19AoVfVt4MSYvWwH7qmBh4Fz2+t/Un3NZSKvr6o6VlXfbdM/AY4w+GaAVd1f8/Q1l2XbX2+J0F/A5iR/luS/JvlHrbaewQfFTpph/h2+3EZ9HcX6eeor7eb2p+yeoVMUq9XLSau9/lMV8MdJHs/gq0IALqqqYzD4IQYuXLXu5u7lTNiPZ8TrK8km4BLgEc6g/XVKX7DC++tNE/pJ/iTJkyNu8x39HQP+RlVdAvwL4PeT/BJjfB3ECvc11/qXra9F9HgX8C5gK4P99dkFepyU1V7/qd5bVZcCVwM3Jfm1VexlMVZ7P54Rr68kbwe+Any8qn4839ARtUn2teL7a+Lfp79UVfXBJSzzCvBKm348yfeBX2HwW3LD0NAlfx3EUvpi/q+jWPavqRi3xyRfAL4+Ro+TsNrrf52qOtrujyf5GoM/rV9Msq6qjrVTAMdXq795elnV/VhVL56cXq3XV5KzGQTrl6rqq6286vtrVF+T2F9vmiP9pUiyNoPv8CfJO4EtwLPtz7mfJLkiSYAbgPsn2No+4Pok5yTZ3Pp6lFX4mopTzld+GDh55cVcPU7KGfOVHUl+Mck7Tk4DVzLYT/uAHW3YDib7GjrVXL3sA25oV6VcAbx88rTGJKz266v9fN8NHKmqzw3NWtX9NVdfE9lfK/HO9KRvbefMMDiqfxF4sNX/KXCYwbve3wX+ydAyU22Hfh/4D7RPJ0+irzbvE23dTzN05RCDqwf+os37xAT23e8BTwCH2gtr3UI9TvDfdaL7Yp4+3tleQ99rr6dPtPovAw8Bz7T78yfUz5cZ/On/f9rr68a5emFwWuDzbR8+wdBVZBPqa1VfX8A/ZHAa5BBwsN2uWe39NU9fK76//BoGSerIW/r0jiTp9Qx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JH/C26o7Q7DI9NXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot distribution of temperatures excluding outliers\n",
    "temperature_all.hist(bins=8, range=(-150, 250), grid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           (50.0, 60.0]\n",
       "1          (90.0, 100.0]\n",
       "2          (90.0, 100.0]\n",
       "3           (60.0, 70.0]\n",
       "6       (-163.001, 20.0]\n",
       "              ...       \n",
       "7664    (-163.001, 20.0]\n",
       "7665    (-163.001, 20.0]\n",
       "7666    (-163.001, 20.0]\n",
       "7667    (-163.001, 20.0]\n",
       "7669    (-163.001, 20.0]\n",
       "Name: Temperature (Reaction Details) [C], Length: 7280, dtype: category\n",
       "Categories (8, interval[float64]): [(-163.001, 20.0] < (20.0, 25.0] < (25.0, 50.0] < (50.0, 60.0] < (60.0, 70.0] < (70.0, 90.0] < (90.0, 100.0] < (100.0, 44124.0]]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create as close to evenly distributed bins as possible (heavy\n",
    "# population of 20 will make challenging or impossible), set bins to 11\n",
    "# as ultimately gave 8 bins when dropping duplicates \n",
    "temperature_tiles = pd.qcut(temperature_all, 11, duplicates='drop')\n",
    "temperature_tiles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create temperature dictionary from quantiles found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>count</th>\n",
       "      <th>frequency</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(-163.001, 20.0]</td>\n",
       "      <td>2933</td>\n",
       "      <td>0.402885</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(70.0, 90.0]</td>\n",
       "      <td>720</td>\n",
       "      <td>0.098901</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(50.0, 60.0]</td>\n",
       "      <td>641</td>\n",
       "      <td>0.088049</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(90.0, 100.0]</td>\n",
       "      <td>629</td>\n",
       "      <td>0.086401</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(100.0, 44124.0]</td>\n",
       "      <td>617</td>\n",
       "      <td>0.084753</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(60.0, 70.0]</td>\n",
       "      <td>588</td>\n",
       "      <td>0.080769</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(20.0, 25.0]</td>\n",
       "      <td>579</td>\n",
       "      <td>0.079533</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(25.0, 50.0]</td>\n",
       "      <td>573</td>\n",
       "      <td>0.078709</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              label  count  frequency category\n",
       "0  (-163.001, 20.0]   2933   0.402885        t\n",
       "1      (70.0, 90.0]    720   0.098901        t\n",
       "2      (50.0, 60.0]    641   0.088049        t\n",
       "3     (90.0, 100.0]    629   0.086401        t\n",
       "4  (100.0, 44124.0]    617   0.084753        t\n",
       "5      (60.0, 70.0]    588   0.080769        t\n",
       "6      (20.0, 25.0]    579   0.079533        t\n",
       "7      (25.0, 50.0]    573   0.078709        t"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# combine bins into single list with counts and frequencies\n",
    "temperature_list = list(zip(temperature_tiles.value_counts().index,\n",
    "                            temperature_tiles.value_counts(),\n",
    "                            temperature_tiles.value_counts(normalize=True)\n",
    "                           )\n",
    "                       )\n",
    "                       \n",
    "# create temperature dataframe for processing\n",
    "temperature_columns = ['label', 'count', 'frequency']\n",
    "temperature_df = pd.DataFrame(temperature_list, columns=temperature_columns)\n",
    "\n",
    "# add column for category label 't'\n",
    "temperature_df['category'] = 't'\n",
    "\n",
    "# visualize dataframe\n",
    "temperature_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine all condition dictionaries to single dictinoary describing the full label space\n",
    "We again trim the label space for each sub-dictionary to avoid categorical sparsity (besides temperature). This is up to user preference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  app.launch_new_instance()\n",
      "/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:4133: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'metal_df':    category                                              label  count  \\\n",
       " 0         m          tetrakis(triphenylphosphine) palladium(0)   1898   \n",
       " 1         m          tris-(dibenzylideneacetone)dipalladium(0)    449   \n",
       " 2         m      bis-triphenylphosphine-palladium(II) chloride    415   \n",
       " 3         m                                palladium diacetate    356   \n",
       " 4         m  (1,1'-bis(diphenylphosphino)ferrocene)palladiu...    332   \n",
       " 5         m             bis(dibenzylideneacetone)-palladium(0)    331   \n",
       " 6         m              bis(tri-t-butylphosphine)palladium(0)    248   \n",
       " 8         m  dichloro(1,1'-bis(diphenylphosphanyl)ferrocene...    217   \n",
       " 13        m  dichloro{bis[1-(dicyclohexylphosphanyl)piperid...    168   \n",
       " 17        m          tris(dibenzylideneacetone)dipalladium (0)    103   \n",
       " 18        m   bis(tricyclohexylphosphine)nickel(II) dichloride    101   \n",
       " 19        m  palladium(l) tri-tert-butylphosphine iodide dimer    101   \n",
       " 20        m  catalyst 1 (aliphatic, phosphine based pincer ...     87   \n",
       " 22        m  1,3-bis[(diphenylphosphino)propane]dichloronic...     62   \n",
       " 24        m                  bis(1,5-cyclooctadiene)nickel (0)     56   \n",
       " 26        m                                  nickel dichloride     56   \n",
       " 29        m  tris(dibenzylideneacetone)dipalladium(0) chlor...     46   \n",
       " 30        m         dichlorobis(tri-O-tolylphosphine)palladium     45   \n",
       " 36        m                                       C20H20ClN3Ni     42   \n",
       " 37        m  dichloro(1,3-bis(2,6-bis(3-pentyl)phenyl)imida...     39   \n",
       " 38        m         bis(triphenylphosphine)nickel(II) chloride     38   \n",
       " 41        m                                          palladium     37   \n",
       " 45        m                              C26H24ClN2NiP*0.1C7H8     35   \n",
       " 46        m                                cobalt(II) chloride     34   \n",
       " 50        m                                  copper(I) bromide     31   \n",
       " 51        m  [1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...     31   \n",
       " 52        m  [1,1'-bis(diphenylphosphino)-ferrocene]palladi...     30   \n",
       " 53        m                                      C40H55Cl5N3Pd     30   \n",
       " 57        m  [1,3-bis(2,6-diisoheptylphenyl)-4,5-dichloroim...     29   \n",
       " 58        m           dichloro bis(acetonitrile) palladium(II)     29   \n",
       " 59        m  1,2-bis(diphenylphosphino)ethane nickel(II) ch...     28   \n",
       " 61        m                     palladium(II) trifluoroacetate     27   \n",
       " 62        m  [1,3-bis(2,6-diisopropylphenyl)imidazol-2-ylid...     26   \n",
       " 63        m                                     C27H22Cl2N3NiP     24   \n",
       " 65        m                          C21H18N8Ni2O(2+)*2F6P(1-)     24   \n",
       " 66        m                 Ni(Cl){2-(Ph2P)C6H4NCH(Ph)P(O)Ph2}     24   \n",
       " 67        m  palladium bis[bis(diphenylphosphino)ferrocene]...     23   \n",
       " \n",
       "     frequency  cumulative  \n",
       " 0    0.318884    0.318884  \n",
       " 1    0.075437    0.394321  \n",
       " 2    0.069724    0.464046  \n",
       " 3    0.059812    0.523858  \n",
       " 4    0.055780    0.579637  \n",
       " 5    0.055612    0.635249  \n",
       " 6    0.041667    0.676915  \n",
       " 8    0.036458    0.713374  \n",
       " 13   0.028226    0.741599  \n",
       " 17   0.017305    0.758905  \n",
       " 18   0.016969    0.775874  \n",
       " 19   0.016969    0.792843  \n",
       " 20   0.014617    0.807460  \n",
       " 22   0.010417    0.817876  \n",
       " 24   0.009409    0.827285  \n",
       " 26   0.009409    0.836694  \n",
       " 29   0.007728    0.844422  \n",
       " 30   0.007560    0.851983  \n",
       " 36   0.007056    0.859039  \n",
       " 37   0.006552    0.865591  \n",
       " 38   0.006384    0.871976  \n",
       " 41   0.006216    0.878192  \n",
       " 45   0.005880    0.884073  \n",
       " 46   0.005712    0.889785  \n",
       " 50   0.005208    0.894993  \n",
       " 51   0.005208    0.900202  \n",
       " 52   0.005040    0.905242  \n",
       " 53   0.005040    0.910282  \n",
       " 57   0.004872    0.915155  \n",
       " 58   0.004872    0.920027  \n",
       " 59   0.004704    0.924731  \n",
       " 61   0.004536    0.929267  \n",
       " 62   0.004368    0.933636  \n",
       " 63   0.004032    0.937668  \n",
       " 65   0.004032    0.941700  \n",
       " 66   0.004032    0.945733  \n",
       " 67   0.003864    0.949597  ,\n",
       " 'ligand_df':    category                                              label  count  \\\n",
       " 10        l              1,1'-bis-(diphenylphosphino)ferrocene    198   \n",
       " 11        l  dicyclohexyl-(2',6'-dimethoxybiphenyl-2-yl)-ph...    190   \n",
       " 12        l                                              XPhos    187   \n",
       " 14        l                                 triphenylphosphine    155   \n",
       " 16        l                            trifuran-2-yl-phosphane    124   \n",
       " 21        l  monophosphine 1,2,3,4,5-pentaphenyl-1'-(di-ter...     75   \n",
       " 27        l                            tris-(o-tolyl)phosphine     52   \n",
       " 32        l                                             ruphos     44   \n",
       " 39        l                            tripiperidino-phosphine     37   \n",
       " 40        l  2?-(dicyclohexylphophanyl)-N2,N2,N6,N6-tetrame...     37   \n",
       " 44        l         tri tert-butylphosphoniumtetrafluoroborate     35   \n",
       " 47        l              1,2-bis-(dicyclohexylphosphino)ethane     33   \n",
       " 49        l  4,5-bis(diphenylphos4,5-bis(diphenylphosphino)...     31   \n",
       " 55        l               1,1'-bis(diphenylphosphino)ferrocene     30   \n",
       " 64        l                N,N,N,N,-tetramethylethylenediamine     24   \n",
       " 71        l                                   [2,2]bipyridinyl     22   \n",
       " 74        l                 4,4'-di-tert-butyl-2,2'-bipyridine     21   \n",
       " 77        l                                           johnphos     20   \n",
       " 78        l  1,2-Ph2-3,4-bis(2,4,6-(t-Bu)3-phenylphophinide...     20   \n",
       " 79        l       pentaphenylferrocenyl di-tert-butylphosphine     20   \n",
       " 82        l                           tri-tert-butyl phosphine     19   \n",
       " 85        l                             tricyclohexylphosphine     18   \n",
       " \n",
       "     frequency  cumulative  \n",
       " 10   0.133603    0.133603  \n",
       " 11   0.128205    0.261808  \n",
       " 12   0.126181    0.387989  \n",
       " 14   0.104588    0.492578  \n",
       " 16   0.083671    0.576248  \n",
       " 21   0.050607    0.626856  \n",
       " 27   0.035088    0.661943  \n",
       " 32   0.029690    0.691633  \n",
       " 39   0.024966    0.716599  \n",
       " 40   0.024966    0.741565  \n",
       " 44   0.023617    0.765182  \n",
       " 47   0.022267    0.787449  \n",
       " 49   0.020918    0.808367  \n",
       " 55   0.020243    0.828610  \n",
       " 64   0.016194    0.844804  \n",
       " 71   0.014845    0.859649  \n",
       " 74   0.014170    0.873819  \n",
       " 77   0.013495    0.887314  \n",
       " 78   0.013495    0.900810  \n",
       " 79   0.013495    0.914305  \n",
       " 82   0.012821    0.927126  \n",
       " 85   0.012146    0.939271  ,\n",
       " 'additive_df':    category                                              label  count  \\\n",
       " 7         a                                   lithium chloride    242   \n",
       " 9         a                                               zinc    205   \n",
       " 15        a                                   copper(l) iodide    153   \n",
       " 23        a                                              water     61   \n",
       " 25        a                        diisobutylaluminium hydride     56   \n",
       " 28        a                                  ammonium chloride     50   \n",
       " 31        a                                     n-butyllithium     45   \n",
       " 33        a                                1-Methylpyrrolidine     42   \n",
       " 34        a                                     sodium formate     42   \n",
       " 35        a                                           Li2CoCl4     42   \n",
       " 42        a                                   hydrogenchloride     36   \n",
       " 43        a                                  caesium carbonate     36   \n",
       " 48        a                                     zinc diacetate     32   \n",
       " 54        a                                      norborn-2-ene     30   \n",
       " 56        a                                potassium carbonate     29   \n",
       " 60        a                                    lithium bromide     28   \n",
       " 69        a  1,3-dimethyl-3,4,5,6-tetrahydro-2(1H)-pyrimidi...     23   \n",
       " 72        a                                methylzinc chloride     22   \n",
       " 73        a                          1-methyl-pyrrolidin-2-one     21   \n",
       " 76        a                                       isoquinoline     20   \n",
       " 80        a                                  zinc(II) chloride     20   \n",
       " 83        a                                   sodium carbonate     19   \n",
       " 87        a                            1-ethyl-2-pyrrolidinone     18   \n",
       " 89        a                                             sodium     16   \n",
       " 90        a                              1-methyl-1H-imidazole     15   \n",
       " 93        a                         tetrabutylammomium bromide     14   \n",
       " 95        a                    oxovanadium(V) ethoxydichloride     12   \n",
       " 98        a                       2-(N,N-dimethylamino)athanol     11   \n",
       " \n",
       "     frequency  cumulative  \n",
       " 7    0.170783    0.170783  \n",
       " 9    0.144672    0.315455  \n",
       " 15   0.107975    0.423430  \n",
       " 23   0.043049    0.466478  \n",
       " 25   0.039520    0.505999  \n",
       " 28   0.035286    0.541284  \n",
       " 31   0.031757    0.573042  \n",
       " 33   0.029640    0.602682  \n",
       " 34   0.029640    0.632322  \n",
       " 35   0.029640    0.661962  \n",
       " 42   0.025406    0.687368  \n",
       " 43   0.025406    0.712773  \n",
       " 48   0.022583    0.735356  \n",
       " 54   0.021171    0.756528  \n",
       " 56   0.020466    0.776994  \n",
       " 60   0.019760    0.796754  \n",
       " 69   0.016231    0.812985  \n",
       " 72   0.015526    0.828511  \n",
       " 73   0.014820    0.843331  \n",
       " 76   0.014114    0.857445  \n",
       " 80   0.014114    0.871560  \n",
       " 83   0.013409    0.884968  \n",
       " 87   0.012703    0.897671  \n",
       " 89   0.011291    0.908963  \n",
       " 90   0.010586    0.919548  \n",
       " 93   0.009880    0.929428  \n",
       " 95   0.008469    0.937897  \n",
       " 98   0.007763    0.945660  ,\n",
       " 'solvent_df':                           label  count  frequency  cumulative category\n",
       " 0               tetrahydrofuran   4599   0.545099    0.545099        s\n",
       " 1        N,N-dimethyl-formamide    900   0.106673    0.651772        s\n",
       " 2     1-methyl-pyrrolidin-2-one    681   0.080716    0.732488        s\n",
       " 3                       toluene    545   0.064596    0.797084        s\n",
       " 4                   1,4-dioxane    335   0.039706    0.836790        s\n",
       " 5        N,N-dimethyl acetamide    248   0.029394    0.866185        s\n",
       " 6                        hexane    214   0.025364    0.891549        s\n",
       " 7                 diethyl ether    204   0.024179    0.915728        s\n",
       " 8                         water    122   0.014460    0.930188        s\n",
       " 9  DMF (N,N-dimethyl-formamide)    102   0.012090    0.942278        s,\n",
       " 'temperature_df':               label  count  frequency category\n",
       " 0  (-163.001, 20.0]   2933   0.402885        t\n",
       " 1      (70.0, 90.0]    720   0.098901        t\n",
       " 2      (50.0, 60.0]    641   0.088049        t\n",
       " 3     (90.0, 100.0]    629   0.086401        t\n",
       " 4  (100.0, 44124.0]    617   0.084753        t\n",
       " 5      (60.0, 70.0]    588   0.080769        t\n",
       " 6      (20.0, 25.0]    579   0.079533        t\n",
       " 7      (25.0, 50.0]    573   0.078709        t}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dataframes for each reagent category for further processing\n",
    "# set category names and labels\n",
    "categories = {'metal': 'm', 'ligand': 'l', 'additive': 'a'}\n",
    "\n",
    "# set open dictionary to store and call category dataframes\n",
    "category_dataframes = {}\n",
    "\n",
    "# iterate categories creating their dataframes and adding to dictionary\n",
    "for category, label in categories.items():\n",
    "    category_dataframes[f'{category}_df'] = nrl_grouped.get_group(label)\n",
    "\n",
    "# filter each dataframe to top 95% by frequency within category\n",
    "for category, dataframe in category_dataframes.items():\n",
    "    cat_df = dataframe\n",
    "    cat_df['frequency'] = cat_df['count'] / cat_df['count'].sum()\n",
    "    cat_df['cumulative'] = cat_df.frequency.cumsum()\n",
    "    cat_df = cat_df[cat_df['cumulative'] <= 0.95]\n",
    "    cat_df.rename(columns={'reagent': 'label'}, inplace=True)\n",
    "    category_dataframes[category] = cat_df\n",
    "\n",
    "# add solvent and temperature dataframes to category dataframes\n",
    "category_dataframes['solvent_df'] = solvent_df\n",
    "category_dataframes['temperature_df'] = temperature_df\n",
    "\n",
    "# visualize dataframe\n",
    "category_dataframes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final dictionary construction\n",
    "A single series is constructed in the desired order, which will define the ultimate multi-label output vector in our predictive models. We apply this dictionary to the reaction data to construct ground truth reagent/condition vectors, which are used as outputs when training our models. This label processing is included in our preprocessor scripts in the main and dataset directories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tetrakis(triphenylphosphine) palladium(0)</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tris-(dibenzylideneacetone)dipalladium(0)</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bis-triphenylphosphine-palladium(II) chloride</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>palladium diacetate</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(1,1'-bis(diphenylphosphino)ferrocene)palladiu...</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>sodium</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>1-methyl-1H-imidazole</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>tetrabutylammomium bromide</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>oxovanadium(V) ethoxydichloride</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>2-(N,N-dimethylamino)athanol</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>105 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 label category\n",
       "0            tetrakis(triphenylphosphine) palladium(0)        m\n",
       "1            tris-(dibenzylideneacetone)dipalladium(0)        m\n",
       "2        bis-triphenylphosphine-palladium(II) chloride        m\n",
       "3                                  palladium diacetate        m\n",
       "4    (1,1'-bis(diphenylphosphino)ferrocene)palladiu...        m\n",
       "..                                                 ...      ...\n",
       "100                                             sodium        a\n",
       "101                              1-methyl-1H-imidazole        a\n",
       "102                         tetrabutylammomium bromide        a\n",
       "103                    oxovanadium(V) ethoxydichloride        a\n",
       "104                       2-(N,N-dimethylamino)athanol        a\n",
       "\n",
       "[105 rows x 2 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set full dictionary vector as combined label space\n",
    "dataframe_order = ['metal_df', 'ligand_df', 'temperature_df', 'solvent_df', 'additive_df']\n",
    "\n",
    "# create series list of all labels and rearrange to desired order\n",
    "dataframe_list = [(category_dataframes[df][['label', 'category']]) for df in dataframe_order]\n",
    "\n",
    "# concatenate category dataframes into single dataframe\n",
    "label_dataframe = pd.concat(dataframe_list, ignore_index=True)\n",
    "\n",
    "# visualize dataframe\n",
    "label_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optionally export dictionary dataframe as csv\n",
    "label_dataframe.to_csv('negishi_dict.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}