docs/source/Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple deduplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from deduplipy.datasets import load_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column names: 'name', 'suburb', 'postcode'\n"
     ]
    }
   ],
   "source": [
    "df_train = load_data(kind='voters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>suburb</th>\n",
       "      <th>postcode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>khimerc thomas</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>2826g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lucille richardst</td>\n",
       "      <td>kannapolis</td>\n",
       "      <td>28o81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                name      suburb postcode\n",
       "0     khimerc thomas   charlotte    2826g\n",
       "1  lucille richardst  kannapolis    28o81"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import `Deduplicator` class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deduplipy.deduplicator import Deduplicator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiate `Deduplicator` class with the column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "myDedupliPy = Deduplicator(['name', 'suburb', 'postcode'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perform the fitting using active learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "myDedupliPy.fit(df_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict on new data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>suburb</th>\n",
       "      <th>postcode</th>\n",
       "      <th>deduplication_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>khimerc thomas</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>2826g</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>chimerc thomas</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28269</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1302</th>\n",
       "      <td>chimerc thmas</td>\n",
       "      <td>chaflotte</td>\n",
       "      <td>28269</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1255</th>\n",
       "      <td>kimberly craddock</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28214</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>kimbefly craddock</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28264</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313</th>\n",
       "      <td>kimberly craddoclc</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>282|4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>l douglas loujdin</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28225</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1139</th>\n",
       "      <td>l douglas loudin</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28205</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>timothy lowder</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28227</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1564</th>\n",
       "      <td>timothy lowder</td>\n",
       "      <td>cbarlotte</td>\n",
       "      <td>282z7</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    name     suburb postcode  deduplication_id\n",
       "0         khimerc thomas  charlotte    2826g                 1\n",
       "1190      chimerc thomas  charlotte    28269                 1\n",
       "1302       chimerc thmas  chaflotte    28269                 1\n",
       "1255   kimberly craddock  charlotte    28214                 4\n",
       "15     kimbefly craddock  charlotte    28264                 4\n",
       "1313  kimberly craddoclc  charlotte    282|4                 4\n",
       "39     l douglas loujdin  charlotte    28225                 7\n",
       "1139    l douglas loudin  charlotte    28205                 7\n",
       "423       timothy lowder  charlotte    28227                 9\n",
       "1564      timothy lowder  cbarlotte    282z7                 9"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = myDedupliPy.predict(df_train)\n",
    "res.sort_values('deduplication_id').head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `Deduplicator` instance can be saved as a pickle file and be applied on new data after training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('myDeduplipy.pkl', 'wb') as f:\n",
    "    pickle.dump(myDedupliPy, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('myDeduplipy.pkl', 'rb') as f:\n",
    "    loaded_obj = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>suburb</th>\n",
       "      <th>postcode</th>\n",
       "      <th>deduplication_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>khimerc thomas</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>2826g</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>chimerc thomas</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28269</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1302</th>\n",
       "      <td>chimerc thmas</td>\n",
       "      <td>chaflotte</td>\n",
       "      <td>28269</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1255</th>\n",
       "      <td>kimberly craddock</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28214</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>kimbefly craddock</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28264</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313</th>\n",
       "      <td>kimberly craddoclc</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>282|4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>l douglas loujdin</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28225</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1139</th>\n",
       "      <td>l douglas loudin</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28205</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>timothy lowder</td>\n",
       "      <td>charlotte</td>\n",
       "      <td>28227</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1564</th>\n",
       "      <td>timothy lowder</td>\n",
       "      <td>cbarlotte</td>\n",
       "      <td>282z7</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    name     suburb postcode  deduplication_id\n",
       "0         khimerc thomas  charlotte    2826g                 1\n",
       "1190      chimerc thomas  charlotte    28269                 1\n",
       "1302       chimerc thmas  chaflotte    28269                 1\n",
       "1255   kimberly craddock  charlotte    28214                 4\n",
       "15     kimbefly craddock  charlotte    28264                 4\n",
       "1313  kimberly craddoclc  charlotte    282|4                 4\n",
       "39     l douglas loujdin  charlotte    28225                 7\n",
       "1139    l douglas loudin  charlotte    28205                 7\n",
       "423       timothy lowder  charlotte    28227                 9\n",
       "1564      timothy lowder  cbarlotte    282z7                 9"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = loaded_obj.predict(df_train)\n",
    "res.sort_values('deduplication_id').head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced deduplication"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load your data. In this example we take a sample dataset that comes with DedupliPy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deduplipy.datasets import load_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column names: 'name', 'suburb', 'postcode'\n"
     ]
    }
   ],
   "source": [
    "df = load_data(kind='voters')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a `Deduplicator` instance and provide advanced settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The similarity metrics per field are entered in a dict. Similarity metric can be any function that takes two strings and output a number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deduplipy.deduplicator import Deduplicator\n",
    "from thefuzz.fuzz import ratio, partial_ratio, token_set_ratio, token_sort_ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "field_info = {'name':[ratio, partial_ratio], 'suburb':[token_set_ratio, token_sort_ratio], 'postcode':[ratio]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We choose our own set of rules for blocking which we define ourselves. We only apply this rule to the 'name' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_two_characters(x):\n",
    "    return x[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- `interaction=True` makes the classifier include interaction features, e.g. `ratio('name') * token_set_ratio('suburb')`. When interaction features are included, the logistic regression classifier applies a L1 regularisation to prevent overfitting.\n",
    "- We set `verbose=1` to get information on the progress and a distribution of scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "myDedupliPy = Deduplicator(field_info=field_info, interaction=True, rules={'name': [first_two_characters]}, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the `Deduplicator` by active learning; enter whether a pair is a match (y) or not (n). When the training is converged, you will be notified and you can finish training by entering 'f'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "myDedupliPy.fit(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the histogram of scores, we decide to ignore all pairs with a similarity probability lower than 0.1 when predicting:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the trained `Deduplicator` on (new) data. The column `deduplication_id` is the identifier for a cluster. Rows with the same `deduplication_id` are found to be the same real world entity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blocking started\n",
      "blocking finished\n",
      "Nr of pairs: 27350\n",
      "scoring started\n",
      "scoring finished\n",
      "Nr of filtered pairs: 954\n",
      "Clustering started\n",
      "Clustering finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>suburb</th>\n",
       "      <th>postcode</th>\n",
       "      <th>deduplication_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lucille richardst</td>\n",
       "      <td>kannapolis</td>\n",
       "      <td>28o81</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1194</th>\n",
       "      <td>lucille richards</td>\n",
       "      <td>kannapolis</td>\n",
       "      <td>28081</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>lutta baldwin</td>\n",
       "      <td>whiteville</td>\n",
       "      <td>28472</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>lutta baldwin</td>\n",
       "      <td>whitevill</td>\n",
       "      <td>28475</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>reb3cca bauerboand</td>\n",
       "      <td>raleigh</td>\n",
       "      <td>27615</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134</th>\n",
       "      <td>rebecca bauerband</td>\n",
       "      <td>raleigh</td>\n",
       "      <td>27615</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1024</th>\n",
       "      <td>rebecca harrell</td>\n",
       "      <td>witnon</td>\n",
       "      <td>27926</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1456</th>\n",
       "      <td>rebecca harrell</td>\n",
       "      <td>winton</td>\n",
       "      <td>27986</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>repecca harrell</td>\n",
       "      <td>winton</td>\n",
       "      <td>27q86</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>675</th>\n",
       "      <td>rebeccah shelton</td>\n",
       "      <td>whittier</td>\n",
       "      <td>28789</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    name      suburb postcode  deduplication_id\n",
       "1      lucille richardst  kannapolis    28o81                 1\n",
       "1194    lucille richards  kannapolis    28081                 1\n",
       "604        lutta baldwin  whiteville    28472                 3\n",
       "995        lutta baldwin   whitevill    28475                 3\n",
       "2     reb3cca bauerboand     raleigh    27615                 5\n",
       "1134   rebecca bauerband     raleigh    27615                 5\n",
       "1024     rebecca harrell      witnon    27926                 7\n",
       "1456     rebecca harrell      winton    27986                 7\n",
       "92       repecca harrell      winton    27q86                 7\n",
       "675     rebeccah shelton    whittier    28789                10"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = myDedupliPy.predict(df, score_threshold=0.1)\n",
    "res.sort_values('deduplication_id').head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}