fuzzy-wuzzy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fuzzywuzzy\n",
    "=======\n",
    "\n",
    "Is a fuzzy string matching library, excellent for categorizing related elements from piles of things.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's generate some strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Strings: ['TNsiqHcCfs', 'hxBFrIoOlD', 'aUUuLDxmEV', 'pOIAJKMuWE', 'VVBwTXpzFM']\n"
     ]
    }
   ],
   "source": [
    "from pwgen import pwgen\n",
    "\n",
    "strs = pwgen(10, 5, capitalize=False, no_numerals=True)\n",
    "print(\"Strings:\", strs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Turn them into a list of similar strings, say translation files with implicit mapping... and shuffle them for a good measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English: ['TNsiqHcCfs_en.pdf', 'VVBwTXpzFM_en.pdf', 'hxBFrIoOlD_en.pdf', 'aUUuLDxmEV_en.pdf', 'pOIAJKMuWE_en.pdf']\n",
      "French: ['TNsiqHcCfs_fr.pdf', 'aUUuLDxmEV_fr.pdf', 'pOIAJKMuWE_fr.pdf', 'VVBwTXpzFM_fr.pdf', 'hxBFrIoOlD_fr.pdf']\n"
     ]
    }
   ],
   "source": [
    "from random import shuffle\n",
    "fr, en = ([x+\"_fr.pdf\" for x in strs],[x+\"_en.pdf\" for x in strs])\n",
    "shuffle(fr)\n",
    "shuffle(en)\n",
    "print('English:', en)\n",
    "print('French:', fr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We can do string manipulation and string comparison to get the right parts, but that's a lot of works, let fuzziness do the job for us"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TNsiqHcCfs_fr.pdf, TNsiqHcCfs_en.pdf ratio 88\n",
      "aUUuLDxmEV_fr.pdf, VVBwTXpzFM_en.pdf ratio 35\n",
      "pOIAJKMuWE_fr.pdf, hxBFrIoOlD_en.pdf ratio 35\n",
      "VVBwTXpzFM_fr.pdf, aUUuLDxmEV_en.pdf ratio 35\n",
      "hxBFrIoOlD_fr.pdf, pOIAJKMuWE_en.pdf ratio 35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alex/.virtualenvs/jupyter/lib/python3.4/site-packages/fuzzywuzzy/fuzz.py:33: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    }
   ],
   "source": [
    "from fuzzywuzzy import fuzz\n",
    "for f, e in zip(fr, en):\n",
    "    print(\"%s, %s ratio %d\" % (f, e, fuzz.ratio(f, e)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Higher ratios have give us some meaningful comparisons, lower -- not really.\n",
    "\n",
    "Let's try a routine that picks the number with highest ratio."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dCJTyXiEQA_en.pdf dCJTyXiEQA_fr.pdf matching ratio 88\n",
      "sjSPylPhsn_en.pdf sjSPylPhsn_fr.pdf matching ratio 88\n",
      "EbMLTThIEj_en.pdf EbMLTThIEj_fr.pdf matching ratio 88\n",
      "BErDwPNGmr_en.pdf BErDwPNGmr_fr.pdf matching ratio 88\n",
      "gmwbXDucaZ_en.pdf gmwbXDucaZ_fr.pdf matching ratio 88\n"
     ]
    }
   ],
   "source": [
    "def get_matching(cmp, in_list):\n",
    "    out_ratio = 0\n",
    "    out_item = None\n",
    "    for item in in_list:\n",
    "        fuzz_ratio = fuzz.ratio(cmp, item)\n",
    "        if fuzz_ratio > out_ratio:\n",
    "            out_ratio = fuzz_ratio\n",
    "            out_item = item\n",
    "    \n",
    "    return (out_item, out_ratio)\n",
    "\n",
    "for en_term in en:\n",
    "    matching = get_matching(en_term, fr)\n",
    "    print(\"%s %s matching ratio %d\" % (en_term, matching[0], matching[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sorting based on fuzziness ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sort term: dCJTyXiEQA_en.pdf\n",
      "Sorted by funzziness: [('dCJTyXiEQA_fr.pdf', 88), ('EbMLTThIEj_fr.pdf', 41), ('BErDwPNGmr_fr.pdf', 35), ('gmwbXDucaZ_fr.pdf', 35), ('sjSPylPhsn_fr.pdf', 35)]\n"
     ]
    }
   ],
   "source": [
    "term = en[0]\n",
    "print(\"sort term:\", term)\n",
    "fuzziness_cmp = lambda x: fuzz.ratio(term, x)\n",
    "print(\"Sorted by funzziness:\", [(k, fuzziness_cmp(k)) for k in sorted(fr, key=fuzziness_cmp, reverse=True)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's add some fuzziness to the input data and see how that goes\n",
    "\n",
    "random_modify whill randomly replace 0 to 3 characters with a random lower-case character"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before & after\n",
      "['dCJTyXiEQA_en.pdf', 'sjSPylPhsn_en.pdf', 'EbMLTThIEj_en.pdf', 'BErDwPNGmr_en.pdf', 'gmwbXDucaZ_en.pdf']\n",
      "['dCJqyXiEQA_en.pdf', 'sjSPylPvsn_en.pdf', 'gkMLTThIEj_en.pdf', 'dErDwPNkmr_en.pdf', 'gmwbXDvcaZ_en.pdf']\n"
     ]
    }
   ],
   "source": [
    "from random import choice, randrange\n",
    "import string\n",
    "\n",
    "def random_modify(in_str):\n",
    "    out_str = bytearray(in_str, 'utf-8')\n",
    "    az = bytearray(string.ascii_lowercase, 'utf-8')\n",
    "    for letter in [randrange(0, 10) for x in range(randrange(3))]:\n",
    "        replacement = choice(az)\n",
    "        out_str[letter] = replacement\n",
    "    return out_str.decode('utf-8')\n",
    "\n",
    "print(\"before & after\")\n",
    "print(en)\n",
    "en = [random_modify(x) for x in en]\n",
    "print(en)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's compare again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dCJqyXiEQA_en.pdf dCJTyXiEQA_fr.pdf matching ratio 82\n",
      "sjSPylPvsn_en.pdf sjSPylPhsn_fr.pdf matching ratio 82\n",
      "gkMLTThIEj_en.pdf EbMLTThIEj_fr.pdf matching ratio 76\n",
      "dErDwPNkmr_en.pdf BErDwPNGmr_fr.pdf matching ratio 76\n",
      "gmwbXDvcaZ_en.pdf gmwbXDucaZ_fr.pdf matching ratio 82\n"
     ]
    }
   ],
   "source": [
    "for en_term in en:\n",
    "    matching = get_matching(en_term, fr)\n",
    "    print(\"%s %s matching ratio %d\" % (en_term, matching[0], matching[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's do the same thing to french and compare fuzziness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before & after\n",
      "['EbMLTThIEj_fr.pdf', 'BErDwPNGmr_fr.pdf', 'dCJTyXiEQA_fr.pdf', 'gmwbXDucaZ_fr.pdf', 'sjSPylPhsn_fr.pdf']\n",
      "['EbMLTThIEj_fr.pdf', 'mErDwPNGmr_fr.pdf', 'dCJTfXiEQA_fr.pdf', 'gmwmXDujaZ_fr.pdf', 'siSPylPwsn_fr.pdf']\n"
     ]
    }
   ],
   "source": [
    "print(\"before & after\")\n",
    "print(fr)\n",
    "fr = [random_modify(x) for x in fr]\n",
    "print(fr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### And compare again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dCJqyXiEQA_en.pdf dCJTfXiEQA_fr.pdf matching ratio 76\n",
      "sjSPylPvsn_en.pdf siSPylPwsn_fr.pdf matching ratio 76\n",
      "gkMLTThIEj_en.pdf EbMLTThIEj_fr.pdf matching ratio 76\n",
      "dErDwPNkmr_en.pdf mErDwPNGmr_fr.pdf matching ratio 76\n",
      "gmwbXDvcaZ_en.pdf gmwmXDujaZ_fr.pdf matching ratio 71\n"
     ]
    }
   ],
   "source": [
    "for en_term in en:\n",
    "    matching = get_matching(en_term, fr)\n",
    "    print(\"%s %s matching ratio %d\" % (en_term, matching[0], matching[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}