2. Data Exploration and Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "11c5c8a1",
   "metadata": {},
   "source": [
    "# Whisky Project\n",
    "\n",
    "## 2. Data Exploration and Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a434a0ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\jamie\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\jamie\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package words to\n",
      "[nltk_data]     C:\\Users\\jamie\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\jamie\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>name</th>\n",
       "      <th>username</th>\n",
       "      <th>link</th>\n",
       "      <th>rating</th>\n",
       "      <th>region</th>\n",
       "      <th>price</th>\n",
       "      <th>date</th>\n",
       "      <th>review</th>\n",
       "      <th>reviewlength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20782</th>\n",
       "      <td>4/2/2017 15:48:30</td>\n",
       "      <td>Glenmorangie Lasanta</td>\n",
       "      <td>An_Imperfect_Guy</td>\n",
       "      <td>https://www.reddit.com/r/Scotch/comments/631zg...</td>\n",
       "      <td>80</td>\n",
       "      <td>Highlands</td>\n",
       "      <td>NaN</td>\n",
       "      <td>04/02/17</td>\n",
       "      <td>I'm back again with some reviews of my latest ...</td>\n",
       "      <td>2707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9132</th>\n",
       "      <td>10/17/2018 20:46:06</td>\n",
       "      <td>Bulleit Rye</td>\n",
       "      <td>strasse007</td>\n",
       "      <td>https://old.reddit.com/r/bourbon/comments/9p4t...</td>\n",
       "      <td>78</td>\n",
       "      <td>Rye</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10/17/18</td>\n",
       "      <td>This was a mystery that I recently reviewed as...</td>\n",
       "      <td>2654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12849</th>\n",
       "      <td>2/23/2017 16:13:58</td>\n",
       "      <td>Croftengea 10 2006 Exclusive Malts</td>\n",
       "      <td>throwboats</td>\n",
       "      <td>https://www.reddit.com/r/Scotch/comments/5ry5m...</td>\n",
       "      <td>81</td>\n",
       "      <td>Highlands</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02/04/17</td>\n",
       "      <td>Croftengea is apparently a peated single malt ...</td>\n",
       "      <td>1116</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 timestamp                                name  \\\n",
       "20782    4/2/2017 15:48:30                Glenmorangie Lasanta   \n",
       "9132   10/17/2018 20:46:06                         Bulleit Rye   \n",
       "12849   2/23/2017 16:13:58  Croftengea 10 2006 Exclusive Malts   \n",
       "\n",
       "               username                                               link  \\\n",
       "20782  An_Imperfect_Guy  https://www.reddit.com/r/Scotch/comments/631zg...   \n",
       "9132         strasse007  https://old.reddit.com/r/bourbon/comments/9p4t...   \n",
       "12849        throwboats  https://www.reddit.com/r/Scotch/comments/5ry5m...   \n",
       "\n",
       "      rating     region price      date  \\\n",
       "20782     80  Highlands   NaN  04/02/17   \n",
       "9132      78        Rye   NaN  10/17/18   \n",
       "12849     81  Highlands   NaN  02/04/17   \n",
       "\n",
       "                                                  review  reviewlength  \n",
       "20782  I'm back again with some reviews of my latest ...          2707  \n",
       "9132   This was a mystery that I recently reviewed as...          2654  \n",
       "12849  Croftengea is apparently a peated single malt ...          1116  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import scipy\n",
    "from fuzzywuzzy import process\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import wordnet, stopwords\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('words')\n",
    "nltk.download('omw-1.4')\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "df = pd.read_csv('./data/gathering_complete.csv', index_col = 'Unnamed: 0')\n",
    "df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c815f8f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAAI0CAYAAADFiUWFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA4UklEQVR4nO3debgkVX3/8fcXBhA1MmwiGdTBQOIWV4IYEVGMoBhBg4piBEMkixvGREHND9wxMeIWUQwIxgWRKBIhIALjEpHVFRCZwCAgy8AMCLLD9/fHOc00Tfed23f6nr7T8349Tz3ddU4tp7q6+3667qmqyEwkSZIkzb61xt0ASZIkaU1h+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3NAdFxI4RkRGxZNxt6RURi2rb9ukpn7NthrnfvtkQEa+KiDMj4ua67RkRO467XbMhIpZM8vataQZ9z0iTwPAtjVBEHNUVcjrDXRFxQ0QsjojjI+KdEbFlwzbNj4iDI+LgVutsLSL2qdv4lHG3Za6IiL2ALwPbAesB19bhzmnMu2Of93FGxB0RcVVE/HdEvHx2t2DNEhELJ/0HEqwZ30fSyswbdwOkCXUXsKw+D+BhwEbAHwC7Ae+PiOOAv8/M6/vMfytwMXDVCNoyHzioPj94BMv7NaVtN41gWaOyD/AcYAnwkwHTjPI1XR3sXx8PBd6emXfPcDnXA/fU578H/H4dXlzfw3tm5j2DZm7o/4DbKftZc9d8Rvt9JK12DN/S7PhhZu7YXRAR8ylHIfcBXl6HP42I7TLzyu5pM/Ns4LFNWjqkzHztuNswE3P5NZ0lT6iPR65C8Ab4k8xc0hmJiMcAh1Dev3sAfwN8ehWWPxKZudO42yBJ02G3E6mRzLwxM0/OzD2BXSlH6RYAx423ZZpQ69fHW0a50My8FNgLuLAWrZY/xiRpXAzf0hhk5snAP9bRZ0TEn3fXT3VyYESsVfs4n1H7kt8VEUsj4oKIODIidumadhFwWdd4bx/eg7un7ZzgVPtlfjgifhkRt0bEjf2mm2obI+LPaxuXR8Qt9cS/Vw+Y9r7+rlMs7wGvSW1rUrqcAHy+Z/uWTDV/n3U8NyK+HhHXRMSd9fEbEfG8KebprGthRDwqIj4XEVfW/tGXRcRHIuJhA1+olYiIh9U+sj+tr+MtEfGziHhPRGzQM22/1/GyrjYeNdN2dMvMu4Az6ujjp2j79hFxTNfrcUNEfCfKiaDRM+27ahvPnWrddd6MiOsiYl5X+ZQnXEbEuhHxxoj4fkQsq+25vH5mHtdn+iPq8v61T91fdL2m7+hT/ze1btFU2zIqw25bnadzfsrBEbF2ROxf32O31mV8KyK2Wcl6nxURJ9bpf1fn3z/Kd9R9y++afhHT/D7qWc/6tZ0XR8Rtdd8fExFbT9G23SLipIi4Nsp35LI6/1ci4pVTv6LS7DJ8S+PzOeC6+rxvKB3gP4HPAztS+pH/jtKn/PHA67h/P8pllD67Hdf2DP2Oim4KnAe8HVgIDN1lISL2B05gRShen9Ll5ksR8alhlzeF2yjbcVcd/y33376l011QRLwfOB14KfBwyuv6cGB34LSI+NBKFvFk4MfAX1P2x1qU1+9tdf51ptuWrjZtBfyM0kf2SZTzBwL4Y+D/AT/rCSD3sGLbO67vKhtlP/1OeF57QNs/DHwfeCXlPzy3AxsCO1FOBP1yRHT/DfpyfXx6RPzhFOt9VX382nS700TE5sDZwCeB7YENgDuAR1E+M+dHxMt6ZvtufdyhzyKf0/V8qvrv9qkbqRluW7d5wImUcwMeR3kPbUj579z3I+KZA9b7WuB7wIvq9HdSvoMOZfB/82byffQw4H8pn4FHA0n5jnol8KOI+IM+bfsAcDzwQspn+DbKd9AfAnsCHx/QPqmNzHRwcBjRABxF+eOwaJrTf7lOf2VP+Y61fElP+Q61/G7KCXW/V8sD2BzYG/hIzzwL6zy5krYsqtPdTDmpchdgrVq3VZ/p9hnQ5t9R/hAfDWxW6zYEPtJpB/DqYds46DWZqk1DzL9nV9s+CWxSyzcGPtFV95o+83bqlgOnAU+s5esBf0UJnUk5uXaY99K6wE/rvL8G/owV4Xsn4PJa9wtgvSnatXAG7+Mdp5ofWIfS7SSBn/epf0utuwZ4PbBBLV+fEpqurvUH9sx3Zi0/aEC7NqQEywSe1VO3pJbv2KetZ9e67wDPBNapdZtTwmLnffsHfd6TdwEP7VlmZ7/8lvKDZu2e+qtq/fOGeM3v+wz0bsMU88xo22r9UV3v2xuAVwDr1ronAT+v9Wf3We9ju/bDiZ33SN2/b6qv2fJaf/Cwn/Wez/RyytHynSk/9NYCng1cUeuP7bP8e2rdB6mf5Vq3KfAXwBHDfiYcHEY5jL0BDg6TNDB8+D6w6w/uOl3lO9I/fL+9lv/PEG0a9o/dndQAuZLp9ukp37FrW74NxBSvzyXd9dNp46DXZKo2TWd+Spi9pNZ9ZcC8nR9Jl1F/kHTVdbZ5UAj+ZK0/fcj30l9OtT8oJ1TeWaf5qz71sxK+gccAx3bVv6Onfj7lB9xtwJMHLP+ZwL2UI6HrdpW/qS7zlwPm++uu/RA9dUvoH74783yPrs9YzzSfqdN8qqe88wNn566yjWrbLwS+Ueuf3lW/VS27A1h/iNf8vs9A7zZMMc+qbNtRXevbvs98T++qf1RP3dG1/Ofd+6+r/u1d8x48aDtXsm2L6nS30vXjv6v+L2r97T3voVfU8ouGfd87OLQa7HYijdfyrucbTWP639bHh/f8y36U/iczf7GKy/hQZmaf8g/Ux60o3TTmgqdQ2gPw/gHTvKc+LgS2HTDNRzPzjj7lx9fHJw7Zrj3q4zf77Y/MvIAV/95/xZDLHsY5Ufq+XxMRv6Nc0q9zje+TgY/1TP8XwEOB72TmT/stMDPPpAToDSkhr+OrlKOWfxQRT+sza6fLyTED3l/97F0fP56lr3o/X6qPf9ZT/r36+JyusmdTfrAtGlDfeX5OZt42zTbO1KpsW8f3M/MHvYWZeR7QuQrTfe/d+r2zex39WGb2u278pyhH20fhuMxc3Kf8BErIXo8Vn19Y8R25QUQ8eERtkEbK8C2tXk6jHO18GrAoIl4TEb8/4nWcuYrz30Xpo/kAmXkJpcsBlG2YCzrtWFoD7QNkZvf1wQe1+5wB5Z35Npxhu86YYprTV9KmUdgE2KwO3WHmPcCufX5w/Gl9fF5XaH/AADyyTtd5JDOvo7zHoec8iNq3ecc6+mWmoZ6Q2fmx9Nkp2vL13rZUnT7b/cL1d6dRP2tGsG0dg9630P+9+xhKP2yAB4R2gMy8lXLeyCj0bV/9sdE5Z6a7fWdR/qOyOXBmROwXDW9qJk2H4Vsar+4/GssGTlXV8Pp3lH/pP5ty8uVVUa6qcVhEPHUEbZr2SYoDXD/gaFhH5w/6pqu4nlHptGNlN9/pHAUc1O6bB5TfXh+Hva/CdNrVadPGEfe/esgIbZmZQfl7sQB4K+X9dyDlvIBem9fHB7MitPcb1umarlsnWL+yZ5teWdvwi8z8+TTbvhGl7zyU/vuD2rJJnWb9nvk7AfpPuo6idofrn1D6fD+76z9RnfrOUfHZsqrb1jHofQsr3rvdJwtv0vX8agb7zRR1wxiqfZm5nNJlazml7/pngUsj4uqIODoinvPAxUhtGb6l8frj+njlFP82vp/MPBLYknLC5TcpJ0stBP4WOC8i3rmKbZoLdyschweNuwEDzIl2ZfGbzPwYpa/xusAXI2JBz6Sdvysfz8yYxnBUz/xfp4SqLbj/lUQ6XU6mddS7py0AT51Oe3q2ufOfmnWAZ0a5tONTgIsz85rMvJdy9HdD4I8jYiHlKiN3M+C/PyO0Sts2yTLzJMp35H6U8xN+AzyCck36RRFx+BibJxm+pXGJiHUpV62Ackm2acvMazPz45m5O+UI6baUk78CeF9EPGmUbR3SJnXbBul0k+k+wn7fJeMiYlDY3GBVGzZApx2D/i3fsUXP9LOts55HTTFNp003DNEHepVl5pcp79kNeWA/+c5lDqdq91TLvhn4Vh19FUC9nNy21JNih1jcDaz4MTmj9nD/ft3bU/5uLuqq7+560jmqen5mjvTmRn2MYttmovtSgZsPnGrqulmXmTdl5ucy85WZuYBygvLnavXrI2LXMTZPazjDtzQ+r6dcgxZWnBQ1tHpE8hzKSXBXUj7X23dNcm/nySx2Tei2DuVqFg9Qr1vdCd/nd1Xd2PV8C/r7kynW2dnGmWxfpx0PiYi+J1PW604v6Jl+tnXW89wppunc/KdVm7q9rz6+tue63J1zBnaMiEFdHVamc3R7j3p99D07y86uW92vTP1vUuemPS+cYVu6w3e//twrq58VI9q2mbiUFSc1bt9vgrrfn96vjvbfRwBk5oWZuR/wo1pk9xONjeFbGoOI2Bno3DnvzMw8cZrzDTyinJn3sOJmM+t1Vf226/n8IZq5Kg4c8If1wPp4SWb+pFNYjxIuqaO79c4UERtTujoM0tnG+cM2lNJvt3M1hUFddg6uj0so11VuoXMlkxf268sfEU9gxRVRjm3Upvtk5qmU612vxf1ft69RrnSxIeVGQANFxKCTUE+i/CDbGHgBM+ty0nFUfdwnIqa8ws6A9nSC9DMo15qG+x/5Po9yc5gdWHFC6KyH7+qo+jjTbRta7WrzzTr6lgE3j/p7yhVv+pnV76OV/NcNyvkKcP/vSKkpw7fUSERsEBE7R8RXKOFifcqNIvaYes77+WBEHBcRu0fEfZcmjIjNIuITlH6OCZzaqcvMG1lx8tPrVnEzpuNWSneaIyLi4bV98+sdD/+qTnNwn/k6AfLdEfGSejUHImI7yg1Epvqj2rlKycui55brK1O7a7y7ju4WEZ+sYZ+I2Li+rp3w9+4aPlr4KuXulgDHR8TzOz9oImInyntoHcq2z/g/J6uo8wNyr84VJTLzBlb8yDogIj7XfWQ8yq3Cnx0RhwE/7LfQegWVzlU63kvpMnA3M/uRcQTlaOeDgNMj4vUR0blaBxHxiIjYKyK+S7k5UK8LKV0t1qOcwHdJZt53omGWu2z+kHIi4paUI7t9rwIyhA0iYpMphs57fFW3baY+RLnq0h8D/xURj67re1BEvAE4hPv/N+s+Db6P/i4iTomIV9cr5FDbNr+eD7NjLTplFtYtTc9sXUDcwWFNHFhx44o7KXf3u4bSB/ZWVtx0Iil/oL9K193XepazI/1vCPOxnuXcRDmS1F32zj7Le09Xfeco8xJg/65pFrGSG9VMNV13mykng3a2cxkr7jj3gJt9dM2/IeUa0p3pbq9tTcrNTl7T7zWp83bfce8uyhVClgA/WNlr2lX//q5139On3R8aMF+nfuGA+oWdaWbwftqKFTePScoR5d91jV8O/OFM2rWS9e44nfkpV3DptO+zPXXvrvu/+33X+5peNsWyd+p5X095YykG3GSn1j2cEoi79+8NXe+vznDQgGV/vWuaw/vUv7Or/rxhX+/e98k0hkWrum2s+K46eNjPeq17Xc/+XcaKmz4dy4ob8RzYZ96RfB/12+es+O7pXv7ynrLPDlqmg0OLwSPf0uxYhxWX+tqYEiQvpdwY4l3AY7KcCHT94EX0dSjwZsq/fX9F6eO8HuUI+leBHTLzg33mey/wDsqR1AAeXYf5Q65/WrJcEeMllH+/r0XZ/h9Rbs/+xgHzLKdcI/pwypGxtSgh4pOU61hf2W++Ou8vKTcROZnyg+QRlO0b1H+83zLeTQl836Qc6XxoXf8JwPMz88ApZp8VWW4u8mTK/uu+0c4vKH2un5SZv2rdro4sR30PraP7RMQWXXXvp7T9cModRNcCHkK5esgplLsgPnuKxZ/B/S9lN5MuJ522XEfp47sX5T8GS4Hfq9W/BL5AuVHRIQMW8d0Bz/uVzfYlBu9nBNs20/V+ntLVpvOZW4/yX4I3U/rob1AnvbHP7LP5ffRlyvk0XwUuovwYfyjlvXQC8JLM/JsRrEeascjMcbdBkiRNiNo96nLKFYSem5mLxtsiaW7xyLckSRqlPSnB+7eUO05K6jLsHdckSdIarp68eDNwPHBVZt5br6jyWsoJmQCfzszbBixCWmPZ7USSJA0lIr5I6WcO5UTL31H6bHcuMfod4M8z8/YHzi2t2TzyLUmShvVpSreS7Sl3s5xPueLJz4AvAl+oJ+RK6uGRb0mSJKmRNebI9yabbJILFy4cdzMkSZI04c4777zrM3PTfnVrTPheuHAh55577ribIUmSpAkXEZcPqvNSg5IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDUyb9wN0OxZeMCJY1v3kkN2Hdu6JUmS5iqPfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRuaNuwGaTAsPOHEs611yyK5jWa8kSdJ0eORbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNdI8fEfEWyPigoj4RUR8JSIeFBFbRsRZEbE4Ir4aEevWader44tr/cKu5RxYyy+OiJ1bb4ckSZI0rKbhOyIWAG8GtsnMJwJrA3sCHwYOzcytgOXAvnWWfYHltfzQOh0R8fg63xOAXYBPR8TaLbdFkiRJGtY4up3MA9aPiHnAg4GrgecBx9X6o4Hd6/Pd6ji1fqeIiFp+TGbekZmXAYuBbds0X5IkSZqZpuE7M68CPgL8mhK6bwLOA27MzLvrZFcCC+rzBcAVdd676/Qbd5f3mec+EbFfRJwbEecuXbp09BskSZIkDaF1t5MNKUettwR+H3gIpdvIrMjMwzNzm8zcZtNNN52t1UiSJEnT0rrbyfOByzJzaWbeBXwdeBYwv3ZDAdgCuKo+vwp4JECt3wC4obu8zzySJEnSnNQ6fP8a2C4iHlz7bu8EXAicAexRp9kb+GZ9fkIdp9afnplZy/esV0PZEtgaOLvRNkiSJEkzMm/lk4xOZp4VEccB5wN3Az8GDgdOBI6JiPfXsiPqLEcA/xkRi4FllCuckJkXRMSxlOB+N/CGzLyn5bZIkiRJw2oavgEy8yDgoJ7iS+lztZLMvB14+YDlfAD4wMgbKEmSJM0S73ApSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ10jx8R8T8iDguIn4ZERdFxDMjYqOIODUiLqmPG9ZpIyI+ERGLI+JnEfG0ruXsXae/JCL2br0dkiRJ0rDGceT748DJmflY4MnARcABwGmZuTVwWh0HeCGwdR32Aw4DiIiNgIOAZwDbAgd1ArskSZI0VzUN3xGxAbADcARAZt6ZmTcCuwFH18mOBnavz3cDvpDFj4D5EbE5sDNwamYuy8zlwKnALs02RJIkSZqB1ke+twSWAp+PiB9HxH9ExEOAzTLz6jrNNcBm9fkC4Iqu+a+sZYPK7yci9ouIcyPi3KVLl454UyRJkqThtA7f84CnAYdl5lOB37GiiwkAmZlAjmJlmXl4Zm6Tmdtsuummo1ikJEmSNGOtw/eVwJWZeVYdP44Sxq+t3Umoj9fV+quAR3bNv0UtG1QuSZIkzVlNw3dmXgNcERF/VIt2Ai4ETgA6VyzZG/hmfX4C8Np61ZPtgJtq95RTgBdExIb1RMsX1DJJkiRpzpo3hnW+CfhSRKwLXAq8jvIj4NiI2Be4HHhFnfYk4EXAYuDWOi2ZuSwi3gecU6d7b2Yua7cJkiRJ0vCah+/M/AmwTZ+qnfpMm8AbBiznSODIkTZOkiRJmkXe4VKSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiNDhe+I+GBEPGq2GiNJkiRNsmGPfL8JuDQiToqIl0SER84lSZKkaRo2PG9OuenNZsDxwOURcVBELBh1wyRJkqRJM1T4zsxbMvOzmfl04BnAt4F/Ai6LiG9ExC6z0UhJkiRpEsz49vKZeQ5wTkQcAHwN2A14SURcDvwbcFhm3juaZkrTs/CAE8e27iWH7Dq2dUuSpNXDjPtsR8QfRMS/ABcAzwK+AewFnAl8DPjMKBooSZIkTYqhjnxHxNrAS4G/AZ4LXAscBnw2M39TJzsmIr4PfBjYb4RtlSRJklZrw3Y7uQrYFPge8CrgG5l5d5/pfgz83iq2TZIkSZoow4bvYyl9uS+aaqLMPAtv4CNJkiTdz1DhOzPfPFsNkSRJkibdsHe4fEdEfHJA3Sci4p9G0yxJkiRp8gzbNeR1wM8G1P2k1kuSJEnqY9jw/SjgkgF1lwKPXrXmSJIkSZNr2PB9KzDoVvJbAHesWnMkSZKkyTVs+P4+8E8RsV53YR1/W62XJEmS1Mewlxo8GPgh8KuI+CLlut8LgNcAGwP7jLJxkiRJ0iQZ9lKDP42I5wIfAd5BOXJ+L/AD4C8y86ejb6IkSZI0GYY98k1mng3sEBHrAxsCyzPztpG3TJIkSZowQ4fvjhq4Dd2SJEnSNA0dviPiMcArKJcdfFBPdWbmvqNomCRJkjRphgrfEbE7cCylr/d1PPDSgjmaZkmSJEmTZ9gj3+8DFgF7ZebS0TdHkiRJmlzDhu/HAG8zeEuSJEnDG/YmO7+kXM9bkiRJ0pCGDd9vB95ZT7qUJEmSNISZ3OFyY+CiiLgEWNZTn5n5nFE0TJIkSZo0w4bve4CLZ6MhkiRJ0qQb9vbyO85SOyRJkqSJN2yfb0mSJEkzNHT4jogFEfHRiDg3Ii6LiCfW8v0j4hmjb6IkSZI0GYYK3xHxBODnwF8Cv6HcYn7dWv1o4C0jbZ0kSZI0QYY98v1vwEXAlsDLgOiq+yGw3YjaJUmSJE2cYa92sj3wqsy8JSLW7qm7FnjEaJolSZIkTZ5hj3zfO0XdJsBtq9AWSZIkaaING77PBl43oO4VwP+uWnMkSZKkyTVst5P3Ad+JiG8DXwYSeH5EvAV4KbDDiNsnSZIkTYyhjnxn5neB3SknXB5JOeHyEODZwO6ZedaoGyhJkiRNimGPfJOZJwInRsRWwMOBGzLTW85LkiRJKzF0+O7IzMXA4hG2RZIkSZpoQ4XviHjtyqbJzC/MvDmSJEnS5Br2yPdRA8qz67nhW5IkSepj2PC9ZZ+yjYEXA68GXrPKLZIkSZIm1FDhOzMv71N8OXB+RATwD5QQLkmSJKnHsDfZmcr3gV1HuDxJkiRpoowyfG8H3DLC5UmSJEkTZdirnfy/PsXrAk+kHPX+1CgaJUmSJE2iYU+4PLhP2R2Uft8fAD60qg2SJEmSJtWwJ1yOspuKJEmStEYxTEuSJEmNDNvn+1HDTJ+Zvx6uOZIkSdLkGrbP9xLufzfLlVl7yOVLkiRJE2vY8P13wLuA3wLHAtcCjwBeATyUctLlHaNsoCRJkjQphg3fjwPOB16amfcdAY+I9wLHA4/LzLeOrnmSJEnS5Bj2hMtXAZ/tDt4AdfwzeGt5SZIkaaBhw/dDgU0H1D0ceMiqNUeSJEmaXMOG70XAByPiT7oLI2JbSn/vRaNpliRJkjR5hg3fb6ScUPmjiFgSEWdFxBLgTOD2Wi9JkiSpj2HvcHlZRDwW2AfYDtgc+AUlfB+dmXeNvIWSJEnShBj2aifUgP25OkiSJEmapqHDN0BEPAnYAdiYcvWTayJiK+DazLx5lA2UJEmSJsWwt5dfD/gi8DIgKHe7/G/gGuBfgF8BB4y4jZIkSdJEGPaEyw8Azwf+EtiMEsA7/gfYeUTtkiRJkibOsN1OXgW8OzO/HBFr99RdBiwcSaskSZKkCTTske+NgYumWNZ6q9YcSZIkaXING74vA545oG5b4OJVa44kSZI0uYYN318ADoiIvYB1allGxHOBtwJHjrJxkiRJ0iQZNnz/C3Ai8J/A8lr2A+A7wMmZ+ckRtk2SJEmaKMPe4fIeYM+I+HfKlU0eDtxACd7fnYX2SZIkSRNj2uE7ItYFfgQckJnfBr4/a62SJEmSJtC0u51k5p3AlsDds9ccSZIkaXIN2+f7VOAFs9EQSZIkadINe5OdTwJfjIh5wPHA1ZRbzN8nMy8dTdMkSZKkyTJs+O6cVPkPlEsL9tN750tJkiRJTCN8R8TzgLMz8xbgr+g50i2pWHjAiWNZ75JDdh3LeiVJ0vCmc+T7VMpdLc/OzKMiYi1gEbBvZl4ym42TJEmSJsl0TriMPuPbA783+uZIkiRJk2vYq51IkiRJmiHDtyRJktTIdK92siAiHlOfr91VdmPvhF5qUJIkSepvuuH7uD5lxw+Y1ksNSpIkSX1MJ3y/btZbIUmSJK0BVhq+M/PoFg2RJEmSJp0nXEqSJEmNjCV8R8TaEfHjiPhWHd8yIs6KiMUR8dWIWLeWr1fHF9f6hV3LOLCWXxwRO49jOyRJkqRhjOvI91uAi7rGPwwcmplbAcuBfWv5vsDyWn5onY6IeDywJ/AEYBfg0xHhiZ6SJEma05qH74jYAtgV+I86HsDzWHFFlaOB3evz3eo4tX6nOv1uwDGZeUdmXgYsBrZtsgGSJEnSDI3jyPfHgLcD99bxjYEbM/PuOn4lsKA+XwBcAVDrb6rT31feZx5JkiRpTmoaviPixcB1mXleo/XtFxHnRsS5S5cubbFKSZIkaaDWR76fBbwkIpYAx1C6m3wcmB8RncsebgFcVZ9fBTwSoNZvANzQXd5nnvtk5uGZuU1mbrPpppuOfmskSZKkITQN35l5YGZukZkLKSdMnp6ZewFnAHvUyfYGvlmfn1DHqfWnZ2bW8j3r1VC2BLYGzm60GZIkSdKMTPf28rPtHcAxEfF+4MfAEbX8COA/I2IxsIwS2MnMCyLiWOBC4G7gDZl5T/tmS5IkSdM3tvCdmYuARfX5pfS5Wklm3g68fMD8HwA+MHstlCRJkkbLO1xKkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRG5o27AZJWzcIDThzLepccsutY1itJ0urMI9+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjcwbdwPWBAsPOHHcTZAkSdIc4JFvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjTQN3xHxyIg4IyIujIgLIuIttXyjiDg1Ii6pjxvW8oiIT0TE4oj4WUQ8rWtZe9fpL4mIvVtuhyRJkjQTrY983w28LTMfD2wHvCEiHg8cAJyWmVsDp9VxgBcCW9dhP+AwKGEdOAh4BrAtcFAnsEuSJElzVdPwnZlXZ+b59fnNwEXAAmA34Og62dHA7vX5bsAXsvgRMD8iNgd2Bk7NzGWZuRw4Fdil3ZZIkiRJwxtbn++IWAg8FTgL2Cwzr65V1wCb1ecLgCu6Zruylg0qlyRJkuassYTviHgo8F/A/pn52+66zEwgR7Se/SLi3Ig4d+nSpaNYpCRJkjRjzcN3RKxDCd5fysyv1+Jra3cS6uN1tfwq4JFds29RywaV309mHp6Z22TmNptuuuloN0SSJEka0ryWK4uIAI4ALsrMj3ZVnQDsDRxSH7/ZVf7GiDiGcnLlTZl5dUScAnyw6yTLFwAHttgGScXCA04c27qXHLLr2NYtSdKqaBq+gWcBfwn8PCJ+UsveSQndx0bEvsDlwCtq3UnAi4DFwK3A6wAyc1lEvA84p0733sxc1mQLJEmSpBlqGr4z8wdADKjeqc/0CbxhwLKOBI4cXeskSZKk2eUdLiVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiPzxt0ASRrWwgNOHMt6lxyy61jWK0maHB75liRJkhoxfEuSJEmNGL4lSZKkRgzfkiRJUiOGb0mSJKkRw7ckSZLUiOFbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1Mm/cDZCk1cXCA04c27qXHLLr2NYtSRodj3xLkiRJjRi+JUmSpEYM35IkSVIjhm9JkiSpEcO3JEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviVJkqRGvL28JK0GxnVre29rL0mj5ZFvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhoxfEuSJEmNeKlBSdJA47rEIXiZQ0mTySPfkiRJUiOGb0mSJKkRu51IkuYk7+opaRJ55FuSJElqxCPfkiR18Yi7pNnkkW9JkiSpEcO3JEmS1IjhW5IkSWrEPt+SJK3h7OcutWP4liRpDhjn3UQltWO3E0mSJKkRw7ckSZLUiN1OJEnSGsd+7hoXw7ckSRoL+7lrTWT4liRJamScPzg86j432OdbkiRJasTwLUmSJDVi+JYkSZIaMXxLkiRJjazW4TsidomIiyNicUQcMO72SJIkSVNZbcN3RKwN/DvwQuDxwKsi4vHjbZUkSZI02GobvoFtgcWZeWlm3gkcA+w25jZJkiRJA63O1/leAFzRNX4l8IwxtUWSJEl9eDfR+1udw/dKRcR+wH519JaIuHiWV7kJcP0sr0Oj535bPbnfVk/ut9WT+23184B9Fh8eU0vGZMzb++hBFatz+L4KeGTX+Ba17D6ZeThweKsGRcS5mblNq/VpNNxvqyf32+rJ/bZ6cr+tftxnc9fq3Of7HGDriNgyItYF9gROGHObJEmSpIFW2yPfmXl3RLwROAVYGzgyMy8Yc7MkSZKkgVbb8A2QmScBJ427HV2adXHRSLnfVk/ut9WT+2315H5b/bjP5qjIzHG3QZIkSVojrM59viVJkqTViuF7RLzV/dwREY+MiDMi4sKIuCAi3lLLN4qIUyPikvq4YS2PiPhE3Xc/i4indS1r7zr9JRGx97i2aU0SEWtHxI8j4lt1fMuIOKvun6/WE6yJiPXq+OJav7BrGQfW8osjYucxbcoaIyLmR8RxEfHLiLgoIp7p523ui4i31u/IX0TEVyLiQX7e5p6IODIirouIX3SVjezzFRFPj4if13k+ERHRdgvXPIbvEfBW93PO3cDbMvPxwHbAG+r+OAA4LTO3Bk6r41D229Z12A84DMqXG3AQ5eZN2wIHdb7gNKveAlzUNf5h4NDM3ApYDuxby/cFltfyQ+t01H29J/AEYBfg0/UzqtnzceDkzHws8GTK/vPzNodFxALgzcA2mflEyoUL9sTP21x0FOW17TbKz9dhwOu75utdl0bM8D0a3up+DsnMqzPz/Pr8ZkoQWEDZJ0fXyY4Gdq/PdwO+kMWPgPkRsTmwM3BqZi7LzOXAqfilNKsiYgtgV+A/6ngAzwOOq5P07rfO/jwO2KlOvxtwTGbekZmXAYspn1HNgojYANgBOAIgM+/MzBvx87Y6mAesHxHzgAcDV+Pnbc7JzO8By3qKR/L5qnUPy8wfZTkJ8Atdy9IsMXyPRr9b3S8YU1vUpf5r9KnAWcBmmXl1rboG2Kw+H7T/3K/tfQx4O3BvHd8YuDEz767j3fvgvv1T62+q07vf2toSWAp8vnYX+o+IeAh+3ua0zLwK+Ajwa0rovgk4Dz9vq4tRfb4W1Oe95ZpFhm9NrIh4KPBfwP6Z+dvuuvoL30v9zCER8WLgusw8b9xt0VDmAU8DDsvMpwK/Y8W/wAE/b3NR7XKwG+XH0+8DD8H/NKyW/Hytfgzfo7HSW92rrYhYhxK8v5SZX6/F19Z/sVEfr6vlg/af+7WtZwEviYgllK5bz6P0JZ5f/y0O998H9+2fWr8BcAPut9auBK7MzLPq+HGUMO7nbW57PnBZZi7NzLuAr1M+g37eVg+j+nxdVZ/3lmsWGb5Hw1vdzyG1H+IRwEWZ+dGuqhOAzhneewPf7Cp/bT1LfDvgpvrvvFOAF0TEhvUo0QtqmWZBZh6YmVtk5kLKZ+j0zNwLOAPYo07Wu986+3OPOn3W8j3r1Rm2pJxAdHajzVjjZOY1wBUR8Ue1aCfgQvy8zXW/BraLiAfX78zOfvPztnoYyeer1v02Irar74PXdi1LsyUzHUYwAC8CfgX8H/CucbdnTR6A7Sn/gvsZ8JM6vIjSP/E04BLgO8BGdfqgXK3m/4CfU87+7yzrrygnEC0GXjfubVtTBmBH4Fv1+WMof8wXA18D1qvlD6rji2v9Y7rmf1fdnxcDLxz39kz6ADwFOLd+5o4HNvTzNvcH4D3AL4FfAP8JrOfnbe4NwFco/fLvovynad9Rfr6Abep74P+AT1FvwOgwe4N3uJQkSZIasduJJEmS1IjhW5IkSWrE8C1JkiQ1YviWJEmSGjF8S5IkSY0YviWt9iJin4jIiLixXsO2u25erTt4DO06uK573sqnHp+IWCsiPhYRV0fEvRFx/Cosa8e6zc8fYRNnXUTsHxEvG3c7JE0+w7ekSbIB8I5xN2I1tAfwFuBfKXc5fPt4mzMW+wOGb0mzzvAtaZJ8G3hTRGw27oa0EhHrjWAxj6uPH8vMMzPzVyNY5qyJiHXq3fjmtBHtG0kTxvAtaZK8vz6+e6qJOt1B+pQfFRFLusYX1i4UfxsRH4qIayLi5oj4Yr0t91YRcUpE3BIRiyNi795lVo+LiDMi4tbateO9EXG/79+I2DQiPhMRV0XEHRHxy4jYr2eaTveaHSLiaxFxI3DWSrZ1l4g4MyJui4ibIuL4rlvBU7f34Dp6T13+PlMsb15EvCMiLoyI2yNiaUScHBGP7Zn0wRHxqYi4vg5fjIj5Pct6Y23bstpl6EcRsWvPNJ198PcR8S8R8RvgDmB+fc0+GxG/qq/tFRHx5YhY0KfdT46Ib0TEDfW1uDgiDux6DR4N7FXXlRFxVM+8J0TE8jrv/0bEs3uWf1REXBkRz4yIH0bEbcC/1LpXR8SP6/vktxHx84j4m0GvsaTJNqf7IUrSkK6m3B55/4j4SGZePqLlHggsAvYGHk8JVfcCTwU+B3wE+Dvg8xFxbmZe0DP/8cCRwIeAnYF/rvMfDBARDwN+AKxfyy6r0x0WEetl5id7lvclyi2n92CK7/GI2AU4ETgdeCXwUOC9wA8i4imZeRXwUuDNwD7AM+us/zfFa3EMsDvwMcptrR8E7ABsTrlVecfHgW8Brwb+iPKa3UN5DTsWAv8BLKnb8efAtyLihZl5cs963wWcA+wHrA3cDjyqPh4ILAV+H3gb8L8R8djMvL2+DttS9t9i4K2UW3RvDTypLvulwEnAT1nxQ2RpnfdpwPeBHwOvB24F/hb4TkT8aWae19XGDerr8xHgncBtEbE98EXgE8A/UQ56PRaYj6Q107jvb+/g4OCwqgMlOCawFbARcCNwZK2bV+sO7pr+4PL194DlHAUs6RpfWOc9vWe6r9fy13SVbQjcDRzUux7ggJ75PwfcDMyv4/9MCZFb95nuemBez3YeOs3X5Vzgks78tWxL4C7go11l7+/3evRZ3vPq+t88xTQ71mmO7in/VN3GGDDfWnVffRv4Zp99cP6gebumXRt4ZJ3+pV3l3wOuAB48xbxLgC/2KT8NuAhYt2c9FwHH97x3EtitZ/5/BJaN+zPi4OAwdwa7nUiaKJm5DPg34LXd3StW0f/0jHeO8J7Std7lwHWU8Nfr2J7xYyhHoZ9Yx3ehdB+5rHbrmBflCimnABtTjrZ3+8bKGhwRDwGeBnw1M+/uaudlwP8Cz1nZMvp4ASVgfm4a057YM/5zYD3gvv74EfH0iPhWRFxL+eFyF/BnlCPlvY7PzH5dhf4uIn4aEbfUZfy6Vv1RrX8w5STSL2XmrdNod/ey16e8Tl8D7u3aL0E56r9Dzyx3UY72dzsH2LB2u3lxb9cbSWsew7ekSXQosIzSxWIUlveM3zlF+YP6zH/tgPFO3+SHU4LcXT3D12r9xj3zX73yJrMhJST2m/Yayn8IhrUx5SjubdOYdlnP+B318UEAEfFIylHljYA3AX8K/AlwMv1fwwdsR0S8Cfg0JQi/DNgW2K57PZTXYS1KV5NhbUQ5yv3PPHDfvJESqrv/ji7NzHu6F5CZ3wVeTvlR9g1gaUR8JyKehKQ1kn2+JU2czLwlIj5EOQL+r30m6fQFXjcz7+wq7w25o7IZcGnPOMBV9fEGylHztwyY/+Ke8QccAe5jeZ3uEX3qHsEDw/F0XA9sFBHrTzOAT2UXSh/pV2TmfcG4Hqnup9827wmclplv65p/y55pllP61z/gJMxpuLHO++/AF/o2KvPelbSRzDwOOC4iHkrplvNh4OSI2KJnfklrAI98S5pUn6aE2/f3qeuciNnp9kHtDvCns9SWV/SM7wncQumKAeVo72OBX2fmuX2Gm4ddYWb+DjgPeHlErN0pj4hHU7Zz0Qy249uUo+l/PYN5e3VC9l2dgoj4Q0oXkWGWcVdP2eu6R2pXkx8Ar6ndSAa5g3LCa/e8v6OcbPlk4Px++2aItpKZt2Tmt4DPUk5Qna0fe5LmMI98S5pImXlHRLwXOLxP9f8ANwGfi4iDKH2R304JxLPh9bV7wjmUq5j8NeUE0Jtq/aGUq5F8PyIOpRzpfgglkD87M3eb4Xr/mdL3+lsR8WlKP/P3ULb934ZdWGaeERH/BXy0dhs5HViH0mXmxMxcNMTivkPpo/2FiPg3Shh9D6XP9nQPDJ0MvCMi3gmcTTkhdI8+0/0j8F3gzLquK4HHAE/JzDfVaS4Enh0RL6Z0y7k+M5cA/0A5YfOUiDiC0v1lE0p/+rUz84CpGljfg5sBZwC/AbagXF3mJ5m5dJrbKWmCeORb0iT7POVqH/eTmTcCL6Z0KTiWcgnAT1IC0mzYjXIi4QnAayhH49/X1Z6bKEejT6LcofMUyqUJd1uVNmW5XN+ulMvaHQt8hnKVju0z8zczXOyelKu47E7ZniOBJzC9fujdbbsA2Ityfe0TKD9+DqAE3el6L+Uo8lsp/amfRPlx07uucyhH1K+g7OeTKJf96+4HfiDlR8+xlB9JB9d5z6f0Rb+BcrnAb1Muo/jH02zrWZQrthwKnErpcvJdyn6RtAaKPiePS5IkSZoFHvmWJEmSGjF8S5IkSY0YviVJkqRGDN+SJElSI4ZvSZIkqRHDtyRJktSI4VuSJElqxPAtSZIkNWL4liRJkhr5/7B2EgkGt74/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x648 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show histogram of reviewlengths\n",
    "\n",
    "plt.figure(figsize=(12,9))\n",
    "plt.hist(df['reviewlength'], bins=20)\n",
    "plt.xlabel('Number of characters', fontsize=16)\n",
    "plt.ylabel('Frequency', fontsize=16)\n",
    "plt.title('Distribution of Review Lengths', fontsize=24);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ba228a0",
   "metadata": {},
   "source": [
    "I have the raw text extracted from the Reddit comments and submissions, and the task now is to extract separate strings for the 'nose', 'palate' and 'finish' (where they exist) for the particular whisky in the review. What follows initially here is my initial approach which I had to abandon, followed by a more fruitful path.\n",
    "\n",
    "#### Extracting 'Nose', 'Palate' and Finish' - Initial Approach\n",
    "\n",
    "These are the regex expressions I'll need to extract out the tasting notes for nose, palate/taste, and finish respectively. \n",
    "\n",
    "    r\"(?:[Nn]ose)[^\\n]+\"\n",
    "    r\"(?:[Pp]alate)[^\\n]+|(?:[Tt]aste)[^\\n]+\"\n",
    "    r\"(?:[Ff]inish)[^\\n]+\"\n",
    "    \n",
    "The plan is to create a whisky by adjective matrix (i.e. a CountVectoriser object) that looks something like this:\n",
    "\n",
    "|   | Nose_vanilla  | Nose_honey  | ...  | Palate_woody  | Palate_cherry  | ...  | Finish_dry | ... |\n",
    "|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|\n",
    "| Aberlour   | 1  | 0  | ...  | 0  | 1  | ...  | 0  | ... |\n",
    "| Glenlivet  |  0 |  1 | .. |  0 | 0  | ... | 0  | ...| \n",
    "| Glenfarclas  | 0  | 0  | ...  | 0  | 1  | ...  | 1  | ...|\n",
    "| Macallan  | 1  | 0  | ...  | 0  | 0  | ...  | 0  |...|\n",
    "| ...  | ...   | ...  | ...  | ...  | ...  | ...  | ...  |...|\n",
    "\n",
    "I'll try disambiguating the nose, palate and finish first of all, and then try it where it's all lumped together. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "95c3d4d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**Whisky Network Reviews #98-100**\n",
      "\n",
      "To celebrate my 100th whisky network review here, I thought I would report on the vilest whisky that I have ever tried. :)\n",
      "\n",
      "But first a little background – on a recent trip to Zurich, I managed to try a number of Swiss single malts. Appenzeller Säntis Malt (\"Swiss Alpine Whisky\") is an offshoot of a brewery (Brauerei Locher).  A signature feature is their use of old beer barrels for aging. Santis has been distilling since 1999 (when production first became legal in Switzerland), and they are currently one of the largest malt whisky producers there. \n",
      "\n",
      "Three of their current bottlings were available there as 50 mL sample bottles (ranging from ~$8-11 CAD each).  Let's see what they are like.\n",
      "\n",
      "------------\n",
      "\n",
      "**Santis Edition Sigel**\n",
      "\n",
      "Edition Sigel is one of the base expressions available from this distiller. It is exclusively \"matured in small oak beer casks\" (with no finishing) and is bottled at 40% ABV. \n",
      "\n",
      "**Colour**: Light gold, but with a slightly brownish tint (reminding me of beer, naturally enough).\n",
      "\n",
      "**Nose**:  Light nose, no alcohol burn. Sweet and somewhat fruity, with red (stawberrry) licorice and candy apples – indeed, candied is the best descriptor. Some citrus. Oak with a bit of wood spice, maybe some anise. Tobacco. Almost earthy in a way – but seems oddly faint, almost as if watered-down. There is a strange effect of the beer cask aging, adding a slightly skunky note (like beer that has long since passed its expiration date, or where the bottle seal has failed). Certainly unique, it doesn’t quite seem like a malt whisky.\n",
      "\n",
      "**Palate**: Not as fruity as the nose suggested – some pear, with a bit of red licorice. Tobacco. Very sour though. Motor oil? Seems very young, and oddly synthetic tasting.  Makes wonder if this is what “Synthehol” on Star Trek TNG would taste like. Some tongue tingle. That skunkiness from the nose comes back with a vengeance as you swallow – making you wish you hadn’t! This is frankly a bit of a mess, with some definite off-putting notes.\n",
      "\n",
      "**Finish**: Short (fortunately). A strong Aspartame-like artificial note, mixed with pear and sour apple. Makes me want to rinse my mouth out with a better whisky immediately (which is exactly what I did, when I was done with this tasting).\n",
      "\n",
      "I’ll be honest here – I couldn’t finish my standard 1.5 ounce pour of this one. I came back to the rest of the sample bottle a couple of nights later, to see if I had misjudged it. Nope, it was just as bad. And if anything, the nose was even weaker now (which was the best part of this whisky originally, if you could call it that). I’m sorry, but my advice to Santis on this one would be to re-distill it and age it longer, please. Also skip the beer casks, if that is what is producing the unique skunky notes.\n",
      "\n",
      "**Score: 72**\n",
      "\n",
      "-----------\n",
      "\n",
      "**Santis Edition Himmelberg**\n",
      "\n",
      "Like Sigel, Edition Himmelberg received its primary aging in old oak beer casks - but it is a blend of whiskies that were subsequently finished in port, sherry, Merlot and other red wine casks for this final bottling. It is bottled at a slightly higher strength (43% ABV).\n",
      "\n",
      "**Colour**: Slightly darker than Sigel, suggesting the extra wine cask finishing. \n",
      "\n",
      "**Nose**:  Sweet nose, I’m definitely getting both classic sherry and port notes. Strong impression of sugar pie (i.e., baked brown sugar and cream), which is novel. Fresh pear and apple fruit, plus dried figs and raisins. Oranges. Rancio. Touch of cinnamon and baking spices – so, apple pie to join that sugar pie. A slightly off-putting underlying sour note (likely from the beer casks again), with a touch of glue. Still, a much better experience than the Edition Sigel malt – this has a lot more character, and is more substantial.\n",
      "\n",
      "**Palate**: Same pear notes as the nose, with additional caramel sweetness adding to the creamy brown sugar. Vanilla and cinnamon. Tobacco. Lighter than expected, both in terms of flavour and texture – although there is some granularity to the mouthfeel, which I like. The 43% ABV is certainly helping here. Less tongue tingle than Sigel, despite the extra alcohol. Some sourness builds at the end unfortunately, but it is still ok.\n",
      "\n",
      "**Finish**: Medium. That creamy brown sugar sweetness returns, with the lingering baked sugar pie experience (and baking spices too). A bit of dark chocolate, which is new. But there is also a persistent sourness on the finish, which detracts personally.\n",
      "\n",
      "Definitely a much better experience. Note that adding water to Edition Himmelberg dampens the whole experience (which is unusual, as I find water usually accentuates the sweetness - not here). I recommend you sample it neat. While decent, there is still something that doesn’t quite gel for me (i.e., that persistent sour note).  But there are enough interesting notes here to make this one worth trying.\n",
      "\n",
      "**Score: 82**\n",
      "\n",
      "-----------\n",
      "\n",
      "**Santis Edition Dreifaltigkeit**\n",
      "\n",
      "This whisky is supposedly “lightly peated,” but most would agree it packs a heavy smokey punch. Apparently, the malt is smoked in multiple ways – first wood-smoked in beech and oak woods, then re-smoked with local peat from the Appenzell Highmoor. Edition Dreifaltigkeit appears to be the same as the earlier Santis Cask Strength Peated, as these whiskies share the same description and cask strength (52% ABV).\n",
      "\n",
      "**Colour**: Much darker than even Himmelberg, with rich mahogany notes. Reminds me of some of the wine cask-aged tropical malts, like Kavalan Solist Sherry Cask or Amrut Portonova.\n",
      "\n",
      "**Nose**:  Oh dear Lord, no. Smokey, but in an extinguished cigarette sort of way, acrid. And very fishy! I could most kindly describe this as Teriyaki-glazed salmon – but only if you are trying to rescue a week-day old salmon fillet by drowning it in soy sauce. Dried nori and tobacco. Asian green tea (specifically, the slightly fishy-smelling kind you get in Japan, not China). The skunkiness from the Sigel is amplified here, with additional fresh glue notes – this is a simply horrible combination of smells. Frankly, I don’t want to put this in my mouth, it literally makes me feel like retching. Oh well, time to take one for the team I guess.\n",
      "\n",
      "**Palate**: Very smokey, but not as acrid as the nose suggested (more wood smoke now instead of cigarette). Sweeter than I expected from the nose too, with brown sugar and a velvety chocolate note that is surprising. BBQ sauce. Tobacco, which is giving it a very bitter aftertaste on the way down. Not much alcohol burn for 52% ABV – you could easily drink this neat (if you were inclined to drink it at all). Not good, but not as bad as the nose indicated.\n",
      "\n",
      "**Finish**: Too long. The fishiness returns, and lingers for a very long time (as bad fish is wont to do). Very smokey. Fortunately, the bitterness fades a bit with time, making this not a completely horrible experience – but still not a good one.  Now you’ll forgive while I go and brush my teeth and tongue …\n",
      "\n",
      "This is a whisky you would be better off drinking with a clothespin over your nose.  At least it is not as horrific in the mouth, with the predominant sweet wood smoke and BBQ notes (think mesquite). With water, the nose is mercifully flattened a little, but it is still an unpleasant experience. Water brings up the sweetness in the mouth slightly, so you will want to try a bit if you want to try this whisky at all.\n",
      "\n",
      "Frankly, I would have scored this below 50 (i.e., undrinkable on the standard liquor reviewer scale), but there are some redeeming virtues on the palate - if you can get past that nose. So, a new low for me personally!\n",
      "\n",
      "**Score: 58**\n",
      "\n",
      "-----------\n",
      "\n",
      "**Personal lifetime scoring stats**:  Mean score:  85.0, Standard Deviation: 5.3, Min score: 58, Max score: 96, Number scored: 217\n",
      "\n",
      "WhiskyAnalysis [Meta-Critic Database](http://whiskyanalysis.com/index.php/database/) for:\n",
      "\n",
      "[Santis Edition Sigel](http://whiskyanalysis.com/index.php/2017/06/24/santis-appenzeller-single-malt-edition-sigel/): 7.93 ± 0.87 on 7 reviews ($$$$)\n",
      "\n",
      "[Santis Edition Himmelberg](http://whiskyanalysis.com/index.php/2017/06/28/santis-appenzeller-single-malt-edition-himmelberg/): too few reviews to score\n",
      "\n",
      "[Santis Edition Dreifaltigkeit](http://whiskyanalysis.com/index.php/2017/06/30/santis-appenzeller-single-malt-edition-dreifaltigkeit/): 7.37 ± 1.67 on 9 reviews ($$$$)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " extracted text for the palate/taste is \n",
      "\n",
      "\n",
      "\n",
      "\n",
      " ['Palate**: Not as fruity as the nose suggested – some pear, with a bit of red licorice. Tobacco. Very sour though. Motor oil? Seems very young, and oddly synthetic tasting.  Makes wonder if this is what “Synthehol” on Star Trek TNG would taste like. Some tongue tingle. That skunkiness from the nose comes back with a vengeance as you swallow – making you wish you hadn’t! This is frankly a bit of a mess, with some definite off-putting notes.', 'Palate**: Same pear notes as the nose, with additional caramel sweetness adding to the creamy brown sugar. Vanilla and cinnamon. Tobacco. Lighter than expected, both in terms of flavour and texture – although there is some granularity to the mouthfeel, which I like. The 43% ABV is certainly helping here. Less tongue tingle than Sigel, despite the extra alcohol. Some sourness builds at the end unfortunately, but it is still ok.', 'Palate**: Very smokey, but not as acrid as the nose suggested (more wood smoke now instead of cigarette). Sweeter than I expected from the nose too, with brown sugar and a velvety chocolate note that is surprising. BBQ sauce. Tobacco, which is giving it a very bitter aftertaste on the way down. Not much alcohol burn for 52% ABV – you could easily drink this neat (if you were inclined to drink it at all). Not good, but not as bad as the nose indicated.', 'palate - if you can get past that nose. So, a new low for me personally!']\n"
     ]
    }
   ],
   "source": [
    "# Testing out regex - extracting palate \n",
    "i = np.random.randint(0,40000)\n",
    "text = df.iloc[i,:]['review']\n",
    "extracted = re.findall(r\"(?:[Pp]alate)[^\\n]+|(?:[Tt]aste)[^\\n]+\", text)\n",
    "print(text)\n",
    "print('\\n\\n\\n\\n\\n extracted text for the palate/taste is \\n\\n\\n\\n\\n', extracted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "341e6ec2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>name</th>\n",
       "      <th>username</th>\n",
       "      <th>link</th>\n",
       "      <th>rating</th>\n",
       "      <th>region</th>\n",
       "      <th>price</th>\n",
       "      <th>date</th>\n",
       "      <th>review</th>\n",
       "      <th>reviewlength</th>\n",
       "      <th>nose</th>\n",
       "      <th>palate</th>\n",
       "      <th>finish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17310</th>\n",
       "      <td>10/27/2017 8:09:16</td>\n",
       "      <td>Glencadam 15</td>\n",
       "      <td>MajorHop</td>\n",
       "      <td>https://www.reddit.com/r/Scotch/comments/79342...</td>\n",
       "      <td>76</td>\n",
       "      <td>Highlands</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10/27/17</td>\n",
       "      <td>**Distilled/Owned by:** Glencadam / Angus Dund...</td>\n",
       "      <td>696</td>\n",
       "      <td>[Nose: Honey, fresh cut flowers, heather, and ...</td>\n",
       "      <td>[Palate/Taste: Vanilla, honey, barley malt, ap...</td>\n",
       "      <td>[Finish: Medium length and warmth. Ginger, nut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31432</th>\n",
       "      <td>8/10/2014 18:37:48</td>\n",
       "      <td>Mortlach 21 1990 Signatory Cask 7708</td>\n",
       "      <td>Scotch_Fanatic</td>\n",
       "      <td>http://www.reddit.com/r/Scotch/comments/2cx8sx...</td>\n",
       "      <td>91</td>\n",
       "      <td>Speyside</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8/7/2014</td>\n",
       "      <td>Hello scotchit! Today i'm reviewing Signatory ...</td>\n",
       "      <td>775</td>\n",
       "      <td>[Nose Juicy black berry, stewed dark fruits, r...</td>\n",
       "      <td>[Palate Sweet juicy black berry, spicy wood, i...</td>\n",
       "      <td>[Finish Tart pineapple and mandarin fades into...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20253</th>\n",
       "      <td>2/1/2013 13:19:08</td>\n",
       "      <td>Glenmorangie 10 Original</td>\n",
       "      <td>CodaRaven</td>\n",
       "      <td>http://www.reddit.com/r/Scotch/comments/17nlef...</td>\n",
       "      <td>72</td>\n",
       "      <td>Highlands</td>\n",
       "      <td>39</td>\n",
       "      <td>1/31/2013</td>\n",
       "      <td>* **Original**\\n\\ncolor: pale gold\\n\\nnose: fl...</td>\n",
       "      <td>3219</td>\n",
       "      <td>[nose: flowery perfume, vanilla, pear apple, n...</td>\n",
       "      <td>[taste: light caramel, plain white rice, mild ...</td>\n",
       "      <td>[finish:  simple, warm, a hint of something - ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp                                  name  \\\n",
       "17310  10/27/2017 8:09:16                          Glencadam 15   \n",
       "31432  8/10/2014 18:37:48  Mortlach 21 1990 Signatory Cask 7708   \n",
       "20253   2/1/2013 13:19:08              Glenmorangie 10 Original   \n",
       "\n",
       "             username                                               link  \\\n",
       "17310        MajorHop  https://www.reddit.com/r/Scotch/comments/79342...   \n",
       "31432  Scotch_Fanatic  http://www.reddit.com/r/Scotch/comments/2cx8sx...   \n",
       "20253       CodaRaven  http://www.reddit.com/r/Scotch/comments/17nlef...   \n",
       "\n",
       "      rating     region price       date  \\\n",
       "17310     76  Highlands   NaN   10/27/17   \n",
       "31432     91   Speyside   NaN   8/7/2014   \n",
       "20253     72  Highlands    39  1/31/2013   \n",
       "\n",
       "                                                  review  reviewlength  \\\n",
       "17310  **Distilled/Owned by:** Glencadam / Angus Dund...           696   \n",
       "31432  Hello scotchit! Today i'm reviewing Signatory ...           775   \n",
       "20253  * **Original**\\n\\ncolor: pale gold\\n\\nnose: fl...          3219   \n",
       "\n",
       "                                                    nose  \\\n",
       "17310  [Nose: Honey, fresh cut flowers, heather, and ...   \n",
       "31432  [Nose Juicy black berry, stewed dark fruits, r...   \n",
       "20253  [nose: flowery perfume, vanilla, pear apple, n...   \n",
       "\n",
       "                                                  palate  \\\n",
       "17310  [Palate/Taste: Vanilla, honey, barley malt, ap...   \n",
       "31432  [Palate Sweet juicy black berry, spicy wood, i...   \n",
       "20253  [taste: light caramel, plain white rice, mild ...   \n",
       "\n",
       "                                                  finish  \n",
       "17310  [Finish: Medium length and warmth. Ginger, nut...  \n",
       "31432  [Finish Tart pineapple and mandarin fades into...  \n",
       "20253  [finish:  simple, warm, a hint of something - ...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def extract_nose(text):\n",
    "    try:\n",
    "        extract_text = re.findall(r\"(?:[Nn]ose)[^\\n]+\", text)\n",
    "        extract_text = [extract_text[i].replace(\"*\", \"\") \n",
    "         for i in range(len(extract_text))]\n",
    "    except:\n",
    "        extract_text = []\n",
    "    return extract_text\n",
    "\n",
    "def extract_palate(text):\n",
    "    try:\n",
    "        extract_text = re.findall(r\"(?:[Pp]alate)[^\\n]+|(?:[Tt]aste)[^\\n]+\", text)\n",
    "        extract_text = [extract_text[i].replace(\"*\", \"\") \n",
    "         for i in range(len(extract_text))]\n",
    "    except:\n",
    "        extract_text = []\n",
    "    return extract_text\n",
    "\n",
    "def extract_finish(text):\n",
    "    try:\n",
    "        extract_text = re.findall(r\"(?:[Ff]inish)[^\\n]+\", text)\n",
    "        extract_text = [extract_text[i].replace(\"*\", \"\") \n",
    "         for i in range(len(extract_text))]\n",
    "    except:\n",
    "        extract_text = []\n",
    "    return extract_text\n",
    "\n",
    "df['nose']=df['review'].apply(extract_nose)\n",
    "df['palate']=df['review'].apply(extract_palate)\n",
    "df['finish']=df['review'].apply(extract_finish)\n",
    "df.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1784fd2",
   "metadata": {},
   "source": [
    "We noticed before that we had reviews that concerned several bottles of whisky at once, and now we're in a position to identify these reviews by counting the size of the lists we've just produced for nose, palate and finish. The modal case should be that every row has one of each, but there's bound to be some discrepancies, and that's what I'll now investigate..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0e7a33f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['noselength']=df['nose'].apply(lambda x: len(x))\n",
    "df['palatelength']=df['palate'].apply(lambda x: len(x))\n",
    "df['finishlength']=df['finish'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "665de978",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14393, 9097)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# more_than_2 is a list of the indices of rows that have 'nose', 'palate/taste', and 'finish'\n",
    "# all at least 2 times, indicating more than one review taking place in all likelihood.\n",
    "# exactly_one shows how many have one extracted bit for nose, palate and finish.\n",
    "\n",
    "more_than_2 = df[(df['noselength']>=2) & (df['palatelength']>=2) & (df['finishlength']>=2)].index.tolist()\n",
    "exactly_one = df[(df['noselength']==1) & (df['palatelength']==1) & (df['finishlength']==1)].index.tolist()\n",
    "\n",
    "len(more_than_2), len(exactly_one)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3382ec63",
   "metadata": {},
   "source": [
    "As you can see, we have a lot of multiple reviews and not many where things are unproblematically one item for nose, taste, and finish. And really I should have realised this earlier, because..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a834121",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10168"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# duplicated is a list of all rows that have a link shared with another row\n",
    "\n",
    "duplicate_links_list = df[df['link'].duplicated()]['link'].tolist()\n",
    "duplicated = df[df['link'].isin(duplicate_links_list)].index.tolist()\n",
    "len(duplicated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "76c7b199",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8798, 5595, 1370)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The set of rows that are in both dataframes\n",
    "both = set(more_than_2) & set(duplicated)\n",
    "\n",
    "# The set of rows that are in more_than_2 but not in duplicated:\n",
    "more_only = set(more_than_2) - set(duplicated)\n",
    "\n",
    "# The set of rows that are in duplicated but not in more_than_2:\n",
    "duplicated_only = set(duplicated) - set(more_than_2)\n",
    "\n",
    "# Check - should add up to 10168+14393 = 24561 (it does)\n",
    "# len(more_only) + len(duplicated_only) + 2*len(both)\n",
    "\n",
    "len(both), len(more_only), len(duplicated_only)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05bbef31",
   "metadata": {},
   "source": [
    "The good news here is that in the 'both' cases, we have 8798 rows where we have different whisky all with the same review text. What needs to happen here is merely that we need to split the text up and match each one with the correct whisky. I say 'merely' - that's probably not going to be that easy.\n",
    "\n",
    "But this means that I have some other cases - the 5595 'more_only' cases. This is where there are two references to taste, nose and finish in the text, but only one whisky. What is happening here is usually that the author has compiled different notes under drinking the whisky neat and drinking it with water. Since they're all valid tasting notes (adding water usually just helps to 'open up' the whisky), the solution here is to simply combine the notes.\n",
    "\n",
    "Finally, I have 1370 cases where there's duplicate reviews, but I haven't picked up on it by looking for 'nose', 'taste' and 'finish'. That's probably just that the review hasn't specifically broken out their review into nose, taste and finish, a problem that is affecting about 1/3 of my dataset. My plan at the moment is that, once I have compiled a matrix of all the adjectives I need, it will be a lot easier to go back into the reviews and simply search for those adjectives directly. It won't be as fine-grained as splitting them up into nose, taste, and finish, and it introduces more noise, so it may not be worth doing, but it's a possibility."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bf802344",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create copy of df to manipulate for the 8798 group\n",
    "df_both = df.loc[list(both), :].copy()\n",
    "\n",
    "# Sort by link\n",
    "df_both = df_both.sort_values('link')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "28dbc841",
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a text file so I can browse through and see if I can figure out a way to \n",
    "# match the reviews\n",
    "\n",
    "def printfile(df, name):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    df - pandas dataframe you want to print out\n",
    "    name - name you want to give the text file\n",
    "    OUTPUTS:\n",
    "    -\n",
    "    '''\n",
    "    with open('./data/' + name, 'w') as file:\n",
    "        for i in range((df.shape[0])):\n",
    "            try:\n",
    "                file.writelines('\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIndex is ' + str(df.index[i]) + '\\n')\n",
    "                file.writelines('Name is ' + str(df.iloc[i,1]) + '\\n')\n",
    "                file.writelines('Review is ' + str(df.iloc[i,8]) + '\\n')\n",
    "            except:\n",
    "                continue\n",
    "                \n",
    "printfile(df_both, 'both.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab869746",
   "metadata": {},
   "source": [
    "Looking through the text file I've just created, it seems as though a good way to split up the reviews that will work for something like 80% of them is by splitting up the review whenever it encounters the '/10(0)' token. This always comes at the end of the review of the particular whisky. The only problem then is that I'll need to match the whisky to the review. This is not so simple, because the name in the 'name' column won't necessarily match exactly. Thus, I'll have to use a bit of 'fuzzy string matching' (using a module called 'fuzzywuzzy') so I'll be able to pick out the review that's most likely to correspond to the correct one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3822576d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting review into a list with various things as separators\n",
    "df_both['review'] = df_both['review'].str.split(r'/100|--|\\*\\*\\*|___|&nbsp')\n",
    "\n",
    "# Deleting small 'reviews' so that it doesn't confuse get_best_fit:\n",
    "def del_small_reviews(reviewlist):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    reviewlist (list) - list of reviews\n",
    "    OUTPUTS:\n",
    "    newlist - reviews with the <100 character ones removed\n",
    "    '''\n",
    "    newlist=[]\n",
    "    for review in reviewlist:\n",
    "        if len(review)>50:\n",
    "            newlist.append(review)\n",
    "    return newlist\n",
    "\n",
    "df_both['review'] = df_both['review'].apply(del_small_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7676f10c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_best_fit(name, reviewlist):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    name (string) - name of the whisky you're trying to identify the review for\n",
    "    reviewlist (list) - list of potential reviews you're matching it to\n",
    "    \n",
    "    OUTPUTS:\n",
    "    output (string) - best guess as to the review for that whisky\n",
    "    '''\n",
    "    bolded_name = r\"\\*\\*\" + re.escape(name) + r\"\\*\\*\" \n",
    "    bolded_match_list = []\n",
    "    unbolded_match_list = []\n",
    "\n",
    "    # Search for exact match on both unbolded and bolded name:\n",
    "    for review in reviewlist:\n",
    "        if re.search(bolded_name, review):\n",
    "            bolded_match_list.append(review)\n",
    "        if re.search(name, review):\n",
    "            unbolded_match_list.append(review)\n",
    "\n",
    "    # If there's only one exact match for bolded name, that's the review you want\n",
    "    if len(bolded_match_list)==1:\n",
    "        output = str(bolded_match_list[0])\n",
    "\n",
    "    # If there's more than one exact match, use fuzzywuzzy to get best match from the matches\n",
    "    elif len(bolded_match_list)>1:\n",
    "        output = str(process.extractOne(name, bolded_match_list)[0])\n",
    "\n",
    "    # If there's no exact matches...\n",
    "    else:\n",
    "        # If there's only one exact match for unbolded name, that's the review you want\n",
    "        if len(unbolded_match_list)==1:\n",
    "            output = str(unbolded_match_list[0])\n",
    "\n",
    "        # If there's more than one exact match, use fuzzywuzzy to get best match from the matches\n",
    "        elif len(unbolded_match_list)>1:\n",
    "            output = str(process.extractOne(name, unbolded_match_list)[0])\n",
    "\n",
    "        # If there's no matches on unbolded names either, just use fuzzywuzzy on original list:\n",
    "        else:\n",
    "            try:\n",
    "                output = str(process.extractOne(name, reviewlist)[0])\n",
    "            except:\n",
    "                output = reviewlist\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6883aa52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seems to work. Run on the whole thing \n",
    "df_both['review'] = df_both.apply(lambda x: get_best_fit(x['name'], x['review']), axis=1)\n",
    "\n",
    "# Print out text file\n",
    "printfile(df_both, 'both2.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f198dc3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract nose, palate and finish from the updated review\n",
    "df_both['nose']=df_both['review'].apply(extract_nose)\n",
    "df_both['palate']=df_both['review'].apply(extract_palate)\n",
    "df_both['finish']=df_both['review'].apply(extract_finish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "957f1ba9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1     3483\n",
       " 0     2514\n",
       " 2     1539\n",
       " 3      569\n",
       " 4      305\n",
       " 5      153\n",
       " 6       84\n",
       " 10      43\n",
       " 8       41\n",
       " 7       38\n",
       " 12      14\n",
       " 9       10\n",
       " 11       5\n",
       " Name: noselength, dtype: int64,\n",
       " 1     3148\n",
       " 0     2327\n",
       " 2     1697\n",
       " 3      639\n",
       " 4      391\n",
       " 6      163\n",
       " 5      151\n",
       " 7       72\n",
       " 8       55\n",
       " 9       54\n",
       " 10      45\n",
       " 12      21\n",
       " 11      18\n",
       " 16      10\n",
       " 23       7\n",
       " Name: palatelength, dtype: int64,\n",
       " 1     3234\n",
       " 0     2495\n",
       " 2     1635\n",
       " 3      686\n",
       " 4      292\n",
       " 5      159\n",
       " 6      103\n",
       " 7       70\n",
       " 8       37\n",
       " 11      30\n",
       " 12      25\n",
       " 9       17\n",
       " 10      10\n",
       " 13       5\n",
       " Name: finishlength, dtype: int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Re-do finding number of instances of 'nose', 'taste'/'palate', and 'finish' in the review\n",
    "df_both['noselength']=df_both['nose'].apply(lambda x: len(x))\n",
    "df_both['palatelength']=df_both['palate'].apply(lambda x: len(x))\n",
    "df_both['finishlength']=df_both['finish'].apply(lambda x: len(x))\n",
    "\n",
    "# Get counts\n",
    "(df_both['noselength'].value_counts(), \n",
    "df_both['palatelength'].value_counts(), \n",
    " df_both['finishlength'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f645563",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of problems still:\n",
    "df_both_prob = df_both[(df_both['noselength']>1) & (df_both['palatelength']>1) & (df_both['finishlength']>1)].index.tolist()\n",
    "df_zeroes = df_both[(df_both['noselength']==0) & (df_both['palatelength']==0) & (df_both['finishlength']==0)].index.tolist()\n",
    "                                                  \n",
    "# As a dataframe\n",
    "df_probs = df_both.loc[df_both_prob,:].sort_values('link')\n",
    "df_zeroes = df_both.loc[df_zeroes,:].sort_values('link')\n",
    "\n",
    "# Print the reviews\n",
    "printfile(df_probs, 'probs.txt')\n",
    "printfile(df_zeroes, 'zeroes.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d2e2f923",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(863, 968)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_probs['link'].unique()), len(df_zeroes['link'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "111d072c",
   "metadata": {},
   "source": [
    "As you can see, there's still lots of problems, but there's just no easy way of cutting up multiple reviews and then identifying the correct one. I'll just have to take the ones that give me exactly one match for nose, taste, and finish, and leave the others, I think. Otherwise, I will have to spend literal weeks trying to get these reviews into the right shape, and that simply isn't a good use of time.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "--------------\n",
    "\n",
    "#### Extracting 'Nose', 'Palate' and Finish' - Subsequent Approach\n",
    "\n",
    "A different approach was needed at this point. What I decided to do was create functions that automate the process of applying different rules to the rows in the 'review' column to separate them into a list of strings that might correspond to individual whiskies, and then progressively take off the rows where I'm relatively confident I've got the right one. \n",
    "\n",
    "So, first of all I need a function that splits the data frame into two, keeping rows that have nose, palate, finish = 1. The idea is that I'm going to progressively strengthen the rules on splitting up the 'review' column until I get as many hits as I can.\n",
    "\n",
    "Note that I've also got a small helper function storage_rule_default() that defines the rule I'm using to decide whether to add a row to my store of successful hits (dfstore). I'll talk more about that in a second."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f7bb3fb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31487, (9097, 16))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dfstore - dataframe that will store rows that have 1 string for 'nose', 'palate', and \n",
    "# for 'finish'\n",
    "dfstore = df[(df['noselength']==1) & (df['palatelength']==1) & (df['finishlength']==1)].copy()\n",
    "\n",
    "# default logical rule for what gets stored in dfstore\n",
    "def storage_rule_default(df):\n",
    "    return df[(df['noselength']==1) & (df['palatelength']==1) & (df['finishlength']==1)]\n",
    "\n",
    "def filter_off_singular_reviews(dfnew, storage_rule=storage_rule_default, dfstore=dfstore):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    dfnew - pandas dataframe containing potential new rows to add to the store of good reviews.\n",
    "    storage_rule - name of function that slices dataframe to get rows that should get stored.\n",
    "    dfstore - dataframe store of singular reviews (i.e. where nose, palate and finish are all \n",
    "            just lists of length one)\n",
    "    OUTPUTS:\n",
    "    dfstore - same as above\n",
    "    remain_idx - indices of the remaining rows\n",
    "    '''\n",
    "    # Make copy of dataframe\n",
    "    df = dfnew.copy()\n",
    "    \n",
    "    # Extract nose, palate and finish from the dataframe\n",
    "    df['nose']=df['review'].apply(extract_nose)\n",
    "    df['palate']=df['review'].apply(extract_palate)\n",
    "    df['finish']=df['review'].apply(extract_finish)\n",
    "    \n",
    "    # Finding number of instances of 'nose', 'taste'/'palate', and 'finish' in the review\n",
    "    df['noselength']=df['nose'].apply(lambda x: len(x))\n",
    "    df['palatelength']=df['palate'].apply(lambda x: len(x))\n",
    "    df['finishlength']=df['finish'].apply(lambda x: len(x))\n",
    "\n",
    "    # Update dfstore and dfremain\n",
    "    dfadd = storage_rule(df)\n",
    "    remain_idx = list(set(df.index) - set(dfadd.index))\n",
    "    dfadd = dfadd.loc[list(set(dfadd.index)-set(dfstore.index)),:]\n",
    "    dfstore = pd.concat([dfstore, dfadd], axis=0)\n",
    "\n",
    "    return remain_idx, dfstore\n",
    "\n",
    "# Run df through - dfstore should remain unchanged here\n",
    "remain_idx, dfstore = filter_off_singular_reviews(df)\n",
    "len(remain_idx), dfstore.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97f3e542",
   "metadata": {},
   "source": [
    "Now I need a function that applies in turn the different values I might use to separate out the 'review' text into a list, and then sends the results to get filtered off. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "04d1820f",
   "metadata": {},
   "outputs": [],
   "source": [
    "rules_list = [\n",
    "    r'___',\n",
    "    r'--',\n",
    "    r'##',\n",
    "    r'~~~',\n",
    "    r'\\*\\*\\*',\n",
    "    r'&nbsp;',\n",
    "    r'/100',\n",
    "    r'/10',\n",
    "    r'Rating',\n",
    "    r'Score',\n",
    "    r'Overall',\n",
    "    r'Total',\n",
    "    r'Review',\n",
    "    r'Conclusion',\n",
    "]\n",
    "\n",
    "def get_singular_reviews(rules_list, storage_rule=storage_rule_default, \n",
    "                         remain_idx=remain_idx, dfstore=dfstore):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    rules_list - list of the expressions to be fed into str.split to split up the \n",
    "    'review' column and then match the right review. \n",
    "    storage_rule - name of function that slices dataframe to get rows that should get stored.\n",
    "    dfstore - dataframe of rows where we're pretty confident we've got the right tasting notes.\n",
    "    remain_idx - indices of the rows from the original df not used.\n",
    "    OUTPUTS:\n",
    "    dfstore - dataframe of rows where we're pretty confident we've got the right tasting notes.\n",
    "    remain_idx - indices of the rows from the original df not used.\n",
    "    '''\n",
    "    \n",
    "    # Apply the rule for splitting the review up\n",
    "    for rule in rules_list:\n",
    "        # Keep track of size of dfstore\n",
    "        numrows_old = dfstore.shape[0]\n",
    "\n",
    "        # Create a df, dfremain to manipulate \n",
    "        dfremain = df.loc[remain_idx,:].copy()\n",
    "\n",
    "        # Applying the rule to split the review\n",
    "        print('Applying the rule {} ...'.format(rule))\n",
    "        dfremain['review'] = dfremain['review'].str.split(rule)\n",
    "\n",
    "        # Delete the small items in the list of 'reviews' so that it doesn't confuse get_best_fit:\n",
    "        dfremain['review'] = dfremain['review'].apply(del_small_reviews)\n",
    "\n",
    "        # Apply get_best_fit to the list of 'reviews' and extract 'nose', etc.\n",
    "        dfremain['review'] = dfremain.apply(lambda x: get_best_fit(x['name'], x['review']), axis=1)\n",
    "\n",
    "        # Drop any rows with null entries for 'nose', etc.\n",
    "        dfremain[['nose', 'palate', 'finish']] = dfremain[['nose', 'palate', 'finish']].dropna()\n",
    "\n",
    "        # Filter off rows with singular reviews for nose, etc.\n",
    "        remain_idx, dfstore = filter_off_singular_reviews(dfremain, storage_rule, dfstore)\n",
    "        print(\"{} rows added.\".format(dfstore.shape[0]-numrows_old))\n",
    "        \n",
    "    return remain_idx, dfstore\n",
    "\n",
    "# Commented out because time-consuming and results are eventually stored and reloaded anyway.\n",
    "\n",
    "#remain_idx, dfstore = get_singular_reviews(rules_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9bb1efc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commented out because time-consuming and results are eventually stored and reloaded anyway.\n",
    "\n",
    "# Print the reviews\n",
    "#printfile(df.loc[remain_idx,:], 'probs.txt')\n",
    "#printfile(dfstore, 'success.txt')\n",
    "#len(remain_idx), dfstore.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5658eacb",
   "metadata": {},
   "source": [
    "Looking through the probs.txt, there's lots of cases where I've actually matched the review just fine, but the rule I'm using for keeping it (i.e. that the length of the lists for nose, palate, and finish must all be 1, is simply too strong). Thus, what I'll do is run the rules list splitting up the reviews again, but this time I'll first allow one of 'taste'/'palate', 'nose', or 'finish' to have a count of 2 to get stored in dfstore, and then I'll run it so that I'll allow the sum of the counts of the lengths to be 3-5 (provided they're all still 1+). So (1, 0, 3), say would be ruled out, as would (2, 2, 2), but (2, 1, 2) or (1, 2, 1) would be fine. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d10010ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use first looser rule that allows one count to be 2:\n",
    "def storage_rule_one_count_can_be_two(df):\n",
    "    return df[(df['noselength']>=1) & \n",
    "               (df['palatelength']>=1) & \n",
    "               (df['finishlength']>=1) &\n",
    "               (df['noselength']+df['palatelength']+df['finishlength']<=4)]\n",
    "\n",
    "#remain_idx, dfstore = get_singular_reviews(rules_list, \n",
    "#                                           storage_rule=storage_rule_one_count_can_be_two)\n",
    "\n",
    "\n",
    "# Use second looser rule that allows two counts to be 2:\n",
    "def storage_rule_two_counts_can_be_two(df):\n",
    "    return df[(df['noselength']>=1) & \n",
    "               (df['palatelength']>=1) & \n",
    "               (df['finishlength']>=1) &\n",
    "               (df['noselength']+df['palatelength']+df['finishlength']<=5)]\n",
    "\n",
    "#remain_idx, dfstore = get_singular_reviews(rules_list, \n",
    "#                                           storage_rule=storage_rule_two_counts_can_be_two)\n",
    "\n",
    "# Print the reviews\n",
    "#printfile(df.loc[remain_idx,:], 'probs.txt')\n",
    "#printfile(dfstore, 'success.txt')\n",
    "#len(remain_idx), dfstore.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5eece1ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((31205, 6), 11194)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dfstore.to_csv('./data/singular_reviews.csv')\n",
    "\n",
    "# Reopening as df, dropping unneeded columns\n",
    "df = pd.read_csv('./data/singular_reviews.csv', index_col = 'Unnamed: 0')\n",
    "df = df[['name', 'rating', 'region', 'nose', 'palate', 'finish']]\n",
    "df.shape, df['name'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8754dc6",
   "metadata": {},
   "source": [
    "I'm relatively happy with that, although 11194 unique whiskies is something I'll need to address. I've got a good amount of data to work with now - about 3/4 of the original dataset. I'm sure I could improve it, but I'm a week in and time is a factor, and it's a case of diminishing marginal returns. Looking through the success.txt, there's some noise in there too because often I'm best-guessing which cut-up review matches which row, but it doesn't appear to be that common. In general, we do seem to be picking out the right bit of the text and matching it to the right whisky.\n",
    "\n",
    "Handily, saving and reloading the file from .csv has also flattened the nose, palate, and finish columns to strings, which is something I'd have needed to do anyway. I can now proceed to start extracting word stems or lemmas from the notes. \n",
    "\n",
    "#### Data Preprocessing\n",
    "\n",
    "Two steps remain before I can proceed with the implementation of a model:\n",
    "\n",
    "1. Address the degree of specificity with the whisky names. It's going to be impossible to train any sort of model unless we group the whiskies by their brand names instead of trying to keep them individuated by their particular expression. The dataset simply isn't large enough to do that.\n",
    "2. Create a vocabulary list. I'll discuss this more below.\n",
    "\n",
    "In terms of the first task, I first tried to match each row of the dataframe on 'name' against a big list of brands. This, however, turned out to be computationally horrendous, so I had to break up the data into distinct sets and go region by region. \n",
    "\n",
    "So what follows is me going through the main regions (stopping short of Sweden) and picking out the main brands of whisky, then matching a 'brand' column from the 'name' for whiskies in that region, and collating them all together into df_collated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e613e21c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Speyside       5872\n",
       "Islay          5832\n",
       "Bourbon        5283\n",
       "Highlands      4106\n",
       "Islands        2126\n",
       "Blend          1656\n",
       "Rye            1308\n",
       "Campbeltown    1156\n",
       "Lowlands        587\n",
       "Canada          571\n",
       "America         564\n",
       "Ireland         543\n",
       "Japan           504\n",
       "India           277\n",
       "Taiwan          173\n",
       "Australia       130\n",
       "Sweden           63\n",
       "France           58\n",
       "Wheat            34\n",
       "Netherlands      33\n",
       "Name: region, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get list of main regions\n",
    "df['region'].value_counts().head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7f09487e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function that matches the brand of whisky to the best one in the list and checks\n",
    "# that the brand name is at least present in the 'name' column.\n",
    "\n",
    "def get_match(string, matchlist):\n",
    "    string = string.replace(\".\", \"\")\n",
    "    string = string.replace(\"\\'\", \"\")\n",
    "    output = str(process.extractOne(string, matchlist)[0])\n",
    "    if re.search(output.lower(), string.lower()): \n",
    "        pass\n",
    "    else:\n",
    "        output = np.NaN\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "19edd210",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Speyside whiskies\n",
    "df_speyside = df[df['region'].str.lower()=='speyside'].copy()\n",
    "df_speyside['region'] = 'Speyside'\n",
    "\n",
    "# list of whiskies in the Speyside region:\n",
    "speyside_list = ['Aberlour', 'Allt-A-Bhainne', 'Ardmore', 'Auchroisk', 'Aultmore', \n",
    "                 'Balmenach','Balvenie', 'BenRiach', 'Benrinnes', 'Benromach', 'Braeval', \n",
    "                 'Caperdonich', 'Cardhu', 'Coleburn', 'Convalmore', 'Cragganmore', \n",
    "                 'Craigellachie', 'Dailuaine', 'Dallas Dhu', 'Dufftown', 'Glen Elgin', \n",
    "                 'Glen Grant', 'Glen Keith', 'Glen Moray', 'Glen Spey', 'Glenallachie', \n",
    "                 'Glenburgie', 'Glendullan', 'Glenfarclas', 'Glenfiddich', 'Glenlivet', \n",
    "                 'Glenlossie', 'Glenrothes', 'Glentauchers', 'Imperial', 'Inchgower',\n",
    "                 'Knockando', 'Linkwood', 'Longmorn', 'Macallan', 'Mannochmore', \n",
    "                 'Miltonduff', 'Mortlach', 'Portknockie', 'Pittyvaich', 'Speyburn',\n",
    "                 'Strathisla', 'Strathmill', 'Tamdhu', 'Tamnavulin', 'Tomintoul', 'Tormore',\n",
    "                'Lismore', 'Stronachie','Old Ballantruan', 'Glenglassaugh']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a7125668",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Islay whiskies\n",
    "df_islay = df[df['region'].str.lower()=='islay'].copy()\n",
    "df_islay['region']='Islay'\n",
    "\n",
    "\n",
    "islay_list = ['Ardbeg', 'Bowmore', 'Bruichladdich', 'Bunnahabhain', 'Caol Ila',\n",
    "              'Finlaggan', 'Islay Storm', 'Kilchoman', 'Lagavulin', 'Laphroaig',\n",
    "              'Macleods', 'Octomore', 'Port Askaig', 'Port Charlotte', 'Port Ellen',\n",
    "              'Ileach']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "77258d01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Bourbon whiskies\n",
    "df_bourbon = df[df['region'].str.lower()=='bourbon'].copy()\n",
    "df_bourbon['region']='Bourbon'\n",
    "\n",
    "# List of Bourbon whiskies to use\n",
    "bourbon_list = ['Old Weller Antique', 'Wild Turkey', 'Bookers', 'Eagle Rare', \n",
    "                'Buffalo Trace', 'Blantons', 'Four Roses', 'WL Weller',\n",
    "                'Henry McKenna', 'Elijah Craig', 'Old Grand Dad', \n",
    "                'Colonel EH Taylor', 'Makers Mark', 'Woodford Reserve',\n",
    "                'Elmer T Lee', 'Bulleit Bourbon', 'Evan Williams', 'Knob Creek',\n",
    "                'Stagg Jr', 'Russells Reserve', 'Blantons', 'Jack Daniels', \n",
    "                'Bakers', 'Orphan Barrel', 'Jim Beam', 'Larceny', 'Noahs Mill',\n",
    "                'Old Rip Van Winkle', 'Angels Envy', '1792', \n",
    "                'Old Ezra', 'Van Winkle', 'Pappy Van Winkle', 'Michters',\n",
    "                'Heaven Hill', 'George T Stagg', 'Rock Hill Farms', 'Basil Haydens',\n",
    "                'Cabin Still', 'Old Crow', 'Red Stag', 'Fighting Cock', 'JTS Brown',\n",
    "                'Old Fitzgerald', 'Vintage Bourbon',  'Rowans Creek', 'Rebel Yell',\n",
    "                'Willett', 'Ezra Brooks', 'Yellowstone', 'Jeffersons', 'Kentucky Gentleman',\n",
    "                'Kentucky Tavern', 'Kentucky Vintage', 'Ten High', 'Very Old Barton',\n",
    "                'Old Charter', 'Old Taylor', 'George Dickel', 'Virginia Gentleman',\n",
    "                'Old Forester']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4ede19f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Highland whiskies\n",
    "df_highland = df[(df['region'].str.lower()=='highlands')|(df['region'].str.lower()=='highland')].copy()\n",
    "df_highland['region']='Highland'\n",
    "\n",
    "# List of highland whiskies to use\n",
    "highland_list = ['Aberfeldy', 'AnCnoc', 'Ardmore', 'Ardnamurchan', 'Balblair', 'Ballechin',\n",
    "                  'Banff', 'Ben Nevis', 'Blair Athol', 'Brora', 'Clynelish', 'Dalmore',\n",
    "                  'Dalwhinnie', 'Deanston', 'Edradour', 'Fettercairn', 'Glen Albyn',\n",
    "                  'Glen Deveron', 'Glen Garioch', 'Glen Mhor', 'Glen Ord', 'Glencadam',\n",
    "                  'Glendronach', 'Glenesk', 'Glenglassaugh', 'Glengoyne', 'Glenlochy',\n",
    "                  'Glenmorangie', 'Glenturret', 'Glenugie', 'Glenury Royal', 'Inchmurrin',\n",
    "                  'Knockdhu', 'Loch Lomond', 'McClelland', 'Millburn', 'North Port', 'Oban',\n",
    "                  'Old Pulteney', 'Royal Brackla', 'Royal Lochnagar', 'Tomatin',\n",
    "                  'Tullibardine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7e21c113",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Islands whiskies\n",
    "df_islands = df[(df['region'].str.lower()=='islands')].copy()\n",
    "df_islands['region']='Islands'\n",
    "\n",
    "# list of islands whiskies to use\n",
    "islands_list = ['Arran', 'Highland Park', 'Jura', 'Ledaig', 'Scapa', 'Talisker', 'Tobermory']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0b51a3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Blend whiskies\n",
    "df_blend = df[(df['region'].str.lower()=='blend')|(df['region'].str.lower()=='blends')].copy()\n",
    "df_blend['region']='Blend'\n",
    "\n",
    "# List of Blend whiskies to use\n",
    "blend_list = ['100 Pipers', 'Antiquary', 'Bailie Nicol Jarvie', 'Ballantines', 'Bells',\n",
    "              'Beneagles', 'Black & White', 'Black Bottle', 'Black Dog', 'Buchanans',\n",
    "              'Chivas Regal', 'Clan MacGregor', 'Cutty Sark', 'Dewars', 'Dimple', \n",
    "              'Famous Grouse', 'Glen Turner', 'Grand Old Parr', 'Grants', 'Haig', \n",
    "              'Hankey Bannister', 'Highland Axe', 'J&B', 'Johnnie Walker', \n",
    "              'Passport Scotch', 'Pattisons whisky', 'Pinch', 'Royal Salute', \n",
    "              'Samuel Dow', 'SIA Scotch Whisky', 'Tè Bheag', 'Te Bheag', 'Té Bheag', \n",
    "              'Teachers', 'Vat 69', 'White Horse', 'Whyte & Mackay', 'Angels Nectar', \n",
    "              'Big Peat', 'Monkey Shoulder', 'Poit Dhubh', 'Rock Oyster', 'Scallywag',\n",
    "              'Sheep Dip', 'Timorous Beastie', 'Glenalmond Everyday', 'Mackinlays Shackleton',\n",
    "              'Pigs Nose', 'Compass Box Hedonism', 'Compass Box Great King', 'Compass Box Asyla',\n",
    "              'Compass Box Spice Tree', 'Compass Box Oak Cross', 'Compass Box Peat Monster',\n",
    "              'Compass Box Orangerie', 'Compass Box Phenomenology', 'Black Grouse',\n",
    "              'Compass Box The Lost Blend', 'Compass Box This Is Not a Luxury Whisky',\n",
    "              'Old Perth', 'Compass Box Flaming Heart', 'Compass Box Rivals',\n",
    "              'Compass Box No Name', 'Compass Box The General', 'Compass Box Enlightenment',\n",
    "              'Compass Box 3 Year Old Deluxe', 'Isle of Skye', \n",
    "              'Old St Andrews Clubhouse', 'Naked Grouse', 'Compass Box Juveniles', \n",
    "              'Epitome 24 1993 Maltbarn', 'Compass Box Double Single', \n",
    "              'Compass Box Eleuthera', 'Compass Box Nectar 10th Anniversary',\n",
    "              'Ron Burgundy Scotch', 'Islay Mist', 'Chivas Century'\n",
    "             ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fbadf5be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Rye whiskies\n",
    "df_rye = df[df['region'].str.lower()=='rye'].copy()\n",
    "df_rye['region']='Rye'\n",
    "\n",
    "# List of Rye whiskies to use\n",
    "rye_list=['Angels Envy Rye', 'Barrell Rye', 'Bookers Rye', 'Bulleit Rye',\n",
    "        'Colonel EH Taylor Straight Rye', 'George Dickel Rye', 'High West Rye',\n",
    "        'Jeffersons 10 Rye', 'Jim Beam Rye', 'Kentucky Owl Rye', 'Knob Creek Rye',\n",
    "        'Michters 10 Rye', 'Michters Barrel Strength Rye', 'Michters Single Barrel Rye',\n",
    "        'New Riff', 'Old Forester Rye', 'Old Overholt', 'Pikesville', 'Redemption',\n",
    "        'Rittenhouse', 'Russells Reserve Rye', 'Russells Reserve Single Barrel Rye',\n",
    "        'Sazerac', 'Smooth Ambler', 'Thomas H Handy', 'Van Winkle Family Reserve Rye',\n",
    "        'WhistlePig', 'Wild Turkey 101 Rye', 'Willett Family Estate Rye', \n",
    "        'Woodford Reserve Rye',  'Sagamore Spirit Rye', 'James E Pepper']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "77e1d2fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Campbeltown whiskies\n",
    "df_campbeltown = df[(df['region'].str.lower()=='campbeltown')].copy()\n",
    "df_campbeltown['region']='Campbeltown'\n",
    "\n",
    "# list of Campbeltown whiskies to use\n",
    "campbeltown_list = ['Campbeltown', 'Glen Scotia', 'Hazelburn', 'Kilkerran', 'Longrow',\n",
    "                'Springbank']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "974454d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Lowland whiskies\n",
    "df_lowland = df[(df['region'].str.lower()=='lowlands')|(df['region'].str.lower()=='lowland')].copy()\n",
    "df_lowland['region']='Lowland'\n",
    "\n",
    "# List of lowland whiskies to use\n",
    "lowland_list = ['Ailsa Bay', 'Annandale', 'Auchentoshan', 'Bladnoch', 'Daftmill',\n",
    "                'Glenflagler', 'Glenkinchie', 'Inchmurrin', 'Inverleven', 'Kinclaith',\n",
    "                'Ladyburn', 'Littlemill', 'Rosebank', 'St Magdalene']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "37f86cba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Canadian whisky dataframe\n",
    "df_canada = df[(df['region'].str.lower()=='canada')].copy()\n",
    "df_canada['region']='Canada'\n",
    "\n",
    "canada_list = ['Alberta', 'Crown Royal', 'Lot No 40', 'JP Wiser', 'Gooderham & Worts',\n",
    "               'Canadian Club', 'Forty Creek', 'Pike Creek', 'Still Waters', \n",
    "               'Glen Breton', 'Pendleton', 'Toronto', 'Shelter Point', 'Mastersons', \n",
    "               'Century Reserve']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "60d8e587",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the American whiskies\n",
    "df_america = df[df['region'].str.lower()=='america'].copy()\n",
    "df_america['region']='America'\n",
    "\n",
    "# List of American whiskies to use\n",
    "america_list=['Balcones', 'High West Campfire', 'Mellow Corn', 'Westland', \"Stranahans\",\n",
    "               'High West Bourye', 'High West Son of Bourye', \"Seagrams\", \"McCarthys\",\n",
    "              'George Dickel', 'Wild Turkey Forgiven', 'Barrell Whiskey', \n",
    "              'Bookers Little Book', 'Bully Boy', 'Cut Spike', 'Corsair', \n",
    "              'Woodford Reserve Straight Malt', 'Jack Daniels', 'Smooth Ambler']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cdf91c0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Irish whiskies\n",
    "df_ireland = df[df['region'].str.lower()=='ireland'].copy()\n",
    "df_ireland['region']='Ireland'\n",
    "\n",
    "# List of Irish whiskies to use\n",
    "ireland_list=['Bushmills', 'Connemara', 'Redbreast', 'Jameson', 'Green Spot', \n",
    "              'Teeling', 'Yellow Spot', 'Knappogue Castle', 'Tullamore Dew', 'Tyrconnell',\n",
    "               \"Writers Tears\", 'Irishman', 'Midleton', 'Powers', 'Greenore', 'Kilbeggan',\n",
    "               'Waterford', 'Blue Spot', 'Hinch', 'Glendalough', 'Cooley']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6aa70509",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Japanese whiskies\n",
    "df_japan = df[df['region'].str.lower()=='japan'].copy()\n",
    "df_japan['region']='Japan'\n",
    "\n",
    "# List of Japanese whiskies to use\n",
    "japan_list=['Nikka', 'Yamazaki', 'Hakushu', 'Hibiki', 'Yoichi', 'Akashi', \n",
    "            'Chita', 'Toki', 'Chichibu', 'Ichiros', 'Karuizawa', 'Kirin Fuji Sanroku',\n",
    "            'Kirin Fuji-Sanroku', 'Mars Komagatake', 'Iwai']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d844db6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Indian whiskies\n",
    "df_india = df[df['region'].str.lower()=='india'].copy()\n",
    "df_india['region']='India'\n",
    "\n",
    "# List of Indian whiskies to use\n",
    "india_list=['Amrut', 'Paul John', 'Adelphi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7ab1dfc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Taiwanese whiskies\n",
    "df_taiwan = df[df['region'].str.lower()=='taiwan'].copy()\n",
    "df_taiwan['region']='Taiwan'\n",
    "\n",
    "# List of Taiwanese whiskies to use\n",
    "taiwan_list=['Kavalan', 'Omar']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "18c36f23",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a smaller dataframe just to get things to work for the Aussie whiskies\n",
    "df_australia = df[df['region'].str.lower()=='australia'].copy()\n",
    "df_australia['region']='Australia'\n",
    "\n",
    "# List of Aussie whiskies to use\n",
    "australia_list=['Heartwood', 'Hellyers Road', 'Sullivans Cove', 'Lime Burners',\n",
    "               'Overeem', 'Bakery Hill', 'Starward', 'Lark', 'Smiths Angaston']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "07dedad6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5633 branded successfully. 246 failed. Time taken: 14 seconds.\n",
      "5679 branded successfully. 159 failed. Time taken: 4 seconds.\n",
      "4233 branded successfully. 1066 failed. Time taken: 15 seconds.\n",
      "3864 branded successfully. 245 failed. Time taken: 8 seconds.\n",
      "2103 branded successfully. 23 failed. Time taken: 1 seconds.\n",
      "1278 branded successfully. 392 failed. Time taken: 5 seconds.\n",
      "846 branded successfully. 462 failed. Time taken: 2 seconds.\n",
      "1151 branded successfully. 6 failed. Time taken: 0 seconds.\n",
      "438 branded successfully. 158 failed. Time taken: 0 seconds.\n",
      "425 branded successfully. 147 failed. Time taken: 0 seconds.\n",
      "365 branded successfully. 200 failed. Time taken: 0 seconds.\n",
      "475 branded successfully. 68 failed. Time taken: 0 seconds.\n",
      "461 branded successfully. 43 failed. Time taken: 0 seconds.\n",
      "272 branded successfully. 5 failed. Time taken: 0 seconds.\n",
      "171 branded successfully. 2 failed. Time taken: 0 seconds.\n",
      "119 branded successfully. 11 failed. Time taken: 0 seconds.\n"
     ]
    }
   ],
   "source": [
    "# Empty dataframe to store correctly-branded rows and incorrectly-branded rows\n",
    "df_collated = pd.DataFrame(columns = ['name', 'rating', 'region', 'nose', 'palate', 'finish', 'brand'])\n",
    "df_fails = pd.DataFrame(columns = ['name', 'rating', 'region', 'nose', 'palate', 'finish', 'brand'])\n",
    "\n",
    "# List containing tuples of the names of the dataframes for each region and the corresponding\n",
    "# list for the whisky names in that region\n",
    "region_df_and_lists_tuples = [(df_speyside, speyside_list),\n",
    "                              (df_islay, islay_list),\n",
    "                              (df_bourbon, bourbon_list),\n",
    "                              (df_highland, highland_list),\n",
    "                              (df_islands, islands_list),\n",
    "                              (df_blend, blend_list),\n",
    "                              (df_rye, rye_list),\n",
    "                              (df_campbeltown, campbeltown_list),\n",
    "                              (df_lowland, lowland_list),\n",
    "                              (df_canada, canada_list),\n",
    "                              (df_america, america_list),\n",
    "                              (df_ireland, ireland_list),\n",
    "                              (df_japan, japan_list),\n",
    "                              (df_india, india_list),\n",
    "                              (df_taiwan, taiwan_list),\n",
    "                              (df_australia, australia_list)]\n",
    "\n",
    "def get_brand_names_per_region(region_df_item, df_collated=df_collated, df_fails=df_fails):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    region_df_item - tuple with the regional pandas dataframe as the [0] position, and\n",
    "    the list of whiskies in that region in the [1] position.\n",
    "    df_collated (pandas dataframe) - points to df_collated by default, stores correct ones\n",
    "    df_fails (pandas dataframe) - points to df_fails, where we've failed.\n",
    "    OUTPUTS:\n",
    "    df_collated (pandas dataframe) - points to df_collated by default, stores correct ones\n",
    "    df_fails (pandas dataframe) - points to df_fails, where we've failed.\n",
    "    '''\n",
    "    df = region_df_item[0]\n",
    "    region_list = region_df_item[1]\n",
    "    \n",
    "    # Apply get_match algorithm to fill 'brand' column. Fails are NaNs\n",
    "    df['brand'] = df['name'].apply(lambda x: get_match(x, region_list))\n",
    "\n",
    "    df_collated = pd.concat([df_collated, df[~df['brand'].isnull()]], axis=0)\n",
    "    df_fails = pd.concat([df_fails, df[df['brand'].isnull()]], axis=0)\n",
    "    \n",
    "    return df_collated, df_fails\n",
    "\n",
    "# Iterate through regions:\n",
    "for region_tuple in region_df_and_lists_tuples:\n",
    "    # counters\n",
    "    start_time = time.time()\n",
    "    collatedrows = df_collated.shape[0]\n",
    "    failsrows = df_fails.shape[0]\n",
    "    \n",
    "    # Apply function\n",
    "    df_collated, df_fails = get_brand_names_per_region(region_tuple, df_collated, df_fails)\n",
    "    \n",
    "    print('{} branded successfully. {} failed. '.format(\n",
    "        df_collated.shape[0] - collatedrows, df_fails.shape[0] - failsrows), end=\"\")\n",
    "    print('Time taken: {} seconds.'.format(round(time.time() - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "7498dace",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27513, 7)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list of corrections to make:\n",
    "brand_corrections = [('1792 Full Proof', '1792'),\n",
    " ('William Larue Weller', 'WL Weller'),\n",
    "('Tè Bheag', 'Té Bheag'),\n",
    "('Te Bheag', 'Té Bheag'),\n",
    "('Michters Barrel Strength Rye','Michters Rye'),\n",
    "(\"Michters 10 Rye\",'Michters Rye'),\n",
    "(\"Michters Single Barrel Rye\",'Michters Rye'),\n",
    "('Russells Reserve Single Barrel Rye','Russells Reserve Rye'),\n",
    "('High West Campfire', 'High West'),\n",
    "('High West Bourye', 'High West'),\n",
    "('High West Son of Bourye', 'High West'),\n",
    "('Kirin Fuji Sanroku', 'Kirin Fuji-Sanroku')   ]\n",
    "\n",
    "for item in brand_corrections:\n",
    "    df_collated.loc[df_collated['brand']==item[0], 'brand'] = item[1]\n",
    "df_collated.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c1631bf",
   "metadata": {},
   "source": [
    "We have 27,513 items where the brand has been identified. We have 366 brands in total, and these will form the rows of what will be a whisky by tasting note matrix. I was initially intending to to be able to match the specific expression of the whisky rather than merely the brand of whisky, but I can see already that I might be in trouble here. 366 brands is already a lot, and I'm concerned that my classifier is going to struggle.\n",
    "\n",
    "The next task we have to do is to build the list of terms that will form the columns of the matrix. What I need is an algorithm that will iterate through each row and look in the 'nose', 'palate' and 'finish' cells in turn, turning a '0' into a '1' in the matrix whenever it detects a match for a particular adjective - i.e. a CountVectoriser with a custom vocabulary. To create the vocabulary list, I'll first have to apply a word lemmatiser and detect adjectives and other useful taste-related terms (like metaphorical nouns or adverbs), and then go through the list manually. Then, once I have this, I'm in a position to construct a pipeline classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ea2289e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenise_and_stem_text(text):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    text (string) - what you want to be lemmatised\n",
    "    OUTPUTS:\n",
    "    lemmas (list) - list of lemmatised next\n",
    "    '''\n",
    "    # Import stopword list and update with a few of my own\n",
    "    stopword_list = stopwords.words(\"english\")\n",
    "    [stopword_list.append(i) for i in ['nose', 'palate', 'taste', 'finish']]\n",
    "    \n",
    "    # Normalise text - remove numbers too as we don't need them\n",
    "    text = re.sub(r\"[^a-zA-Z]\", \" \", text.lower())\n",
    "    \n",
    "    # tokenise\n",
    "    words = text.split()\n",
    "    \n",
    "    # Checks it's a word and removes stop words\n",
    "    words = [word for word in words if word not in stopword_list]\n",
    "    \n",
    "    # Create stemmer object\n",
    "    stemmer = PorterStemmer()\n",
    "    \n",
    "    # Add lemmas\n",
    "    lemmas = []\n",
    "    for word in words:\n",
    "        lemmas.append(stemmer.stem(word))\n",
    "    \n",
    "    return lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b2e2aac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for creating custom vocab list (OMG this took ages!!! 15000 word-stems to look through...)\n",
    "\n",
    "#vectoriser = CountVectorizer(tokenizer=tokenise_and_stem_text)\n",
    "#corpus = pd.concat([df_collated['nose'], df_collated['palate'], df_collated['finish']])\n",
    "#X = vectoriser.fit_transform(corpus)\n",
    "\n",
    "#tasting_notes_list = vectoriser.get_feature_names_out().tolist()\n",
    "\n",
    "#i=15\n",
    "#x = 1000 * i\n",
    "#y = x + 1000\n",
    "#print(tasting_notes_list[x:y])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "084fa749",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1985"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordlist = ['ablaz', 'abnorm', 'abomin', 'abound', 'abras', 'abrupt', 'absinth', 'abuzz',\n",
    "            'acacia', 'accentu', 'access', 'acerb', 'acet', 'aceton', 'acid', 'acidi',\n",
    "            'addict', 'addit', 'adher', 'adhes', 'aerat', 'aflam', 'afterburn',\n",
    "            'aftereffect', 'afterglow', 'aftertas', 'aftertast', 'agreeabl', 'alcohol',\n",
    "            'alcoholburn', 'alcoholi', 'ale', 'alkali', 'alkalin', 'alkaloid', 'allsort',\n",
    "            'allspic', 'allta', 'almond', 'alpin', 'altoid', 'amaretto', 'amaroso',\n",
    "            'ambrosia', 'ameretto', 'americano', 'ammon', 'ammonia', 'ammonium',\n",
    "            'amontillado', 'anaesthetis', 'anchovi', 'angelica', 'anise', 'anisett',\n",
    "            'antifreez', 'antisept', 'appl', 'applecak', 'applejack', 'applejuic',\n",
    "            'applemint', 'applesauc', 'appleskin', 'appleton', 'applewood', 'appley',\n",
    "            'apricot', 'armchair', 'armpit', 'aroma', 'aromaish', 'aromat', 'ash', 'ashen',\n",
    "            'ashphalt', 'ashtray', 'asparagu', 'aspartam', 'asphalt', 'aspirin', 'ass',\n",
    "            'assault', 'assert', 'astonish', 'astound', 'autumn', 'avalanch', 'avocado',\n",
    "            'awak', 'awaken', 'awash', 'bacon', 'baffl', 'bagel', 'baguett', 'bakeappl',\n",
    "            'bakelit', 'baker', 'bakeshop', 'bakewel', 'bamboo', 'banana', 'bananabread',\n",
    "            'band', 'bandag', 'banger', 'banoffe', 'barbecu', 'barbequ', 'barbershop',\n",
    "            'bark', 'barley', 'barli', 'barn', 'barnwood', 'barnyard', 'barnyardi',\n",
    "            'barnyardish', 'basalt', 'basement', 'basil', 'basket', 'basswood', 'bastard',\n",
    "            'bat', 'bath', 'bathroom', 'battenberg', 'batter', 'batteri', 'bazooka', 'bbq',\n",
    "            'bbqed', 'bbqish', 'bbqyness', 'beach', 'beachi', 'beachsid', 'bear',\n",
    "            'bearabl', 'beast', 'beasti', 'beastli', 'beauti', 'bee', 'beebalm', 'beech',\n",
    "            'beechnut', 'beechwood', 'beef', 'beefi', 'beefier', 'beefy', 'beer', 'beeri',\n",
    "            'beerish', 'beeswax', 'beeswaxey', 'beeswaxi', 'beet', 'beetroot', 'bereft',\n",
    "            'berri', 'berry', 'berryfruit', 'big', 'bigger', 'biggest', 'biggish', 'bile',\n",
    "            'bileish', 'billi', 'bin', 'birch', 'biryani', 'biscotti', 'biscuit',\n",
    "            'biscuiti', 'biscuity', 'bite', 'biter', 'bitey', 'biti', 'bitingli', 'bitter',\n",
    "            'bitteress', 'bitteri', 'bitterish', 'bitterli', 'bittermelon', 'bittersweet',\n",
    "            'bitumen', 'black', 'blackberri', 'blackcurr', 'blackpepp', 'blanch', 'blanco',\n",
    "            'bland', 'blander', 'blandest', 'blank', 'blast', 'blaster', 'bleach',\n",
    "            'bleugh', 'bleurgh', 'blitz', 'blizzard', 'block', 'blond', 'bloom', 'blossom',\n",
    "            'blueberri', 'bodied', 'bog', 'boggi', 'boggy', 'bold', 'bolder', 'boldest',\n",
    "            'bombast', 'bomber', 'bombshel', 'bonbon', 'bonfir', 'boost', 'boot', 'booz',\n",
    "            'boozey', 'boozi', 'bore', 'boring', 'borscht', 'bouillon', 'bounc', 'bounci',\n",
    "            'bouquet', 'bourboney', 'bourboni', 'bourbonish', 'boutiqu', 'bovril', 'box',\n",
    "            'boxwood', 'brambl', 'brambleberri', 'bran', 'brandi', 'brandyish', 'brash',\n",
    "            'brasher', 'brashli', 'brasil', 'brass', 'brassi', 'brassier', 'bratwurst',\n",
    "            'braus', 'brawni', 'bread', 'breadcrumb', 'breadfruit', 'breadi', 'breadier',\n",
    "            'breadstick', 'breezey', 'breezi', 'brie', 'brief', 'briefer', 'briefest',\n",
    "            'briefli', 'brighten', 'brighter', 'brighti', 'brightli', 'brillianc',\n",
    "            'brilliant', 'brilliantli', 'brim', 'brimfir', 'brimston', 'brine', 'briney',\n",
    "            'brini', 'brinier', 'brink', 'brioch', 'briquet', 'briquett', 'brisket',\n",
    "            'broad', 'broaden', 'broader', 'broccoli', 'brutal', 'brute', 'brutish',\n",
    "            'bubblegum', 'bubblegummi', 'bubbleyum', 'buckthorn', 'buckwheat', 'burdock',\n",
    "            'burgandi', 'burger', 'burgundi', 'burlap', 'burli', 'burn', 'burner', 'burni',\n",
    "            'burnish', 'burnout', 'burnt', 'burnti', 'burst', 'bursti', 'butter',\n",
    "            'buttercream', 'buttercup', 'butterfat', 'butterfing', 'butterfli',\n",
    "            'buttergeb', 'butteri', 'buttermilk', 'butternut', 'butterscotch',\n",
    "            'butterscotchi', 'buttertart', 'butterworth', 'buttery', 'cacaphoni', 'cacoa',\n",
    "            'cafe', 'cafeteria', 'caff', 'cak', 'cake', 'cakey', 'caki', 'calm', 'calmer',\n",
    "            'calmli', 'camembert', 'campfir', 'campfirey', 'candl', 'candlewax',\n",
    "            'candybar', 'candycan', 'candyfloss', 'candyish', 'cane', 'cannabi',\n",
    "            'cannabisi', 'cantaloup', 'canteloup', 'caocao', 'cappuccino', 'cappucino',\n",
    "            'capuccino', 'caram', 'caramac', 'caramael', 'caramali', 'caramalis',\n",
    "            'carambola', 'caramel', 'caramelis', 'caramelli', 'caramilk', 'cararmel',\n",
    "            'cardamom', 'cardamon', 'cardboard', 'cardboardi', 'carrot', 'cascad',\n",
    "            'cashew', 'casserol', 'caster', 'castor', 'caustic', 'cautious', 'caveman',\n",
    "            'cavern', 'caviar', 'cayann', 'cayen', 'cayenn', 'cedar', 'cedari',\n",
    "            'cedarwood', 'celeri', 'cereal', 'cereali', 'ceris', 'chai', 'chalk',\n",
    "            'chalkboard', 'chalki', 'chalky', 'champignon', 'chanterel', 'chantilli',\n",
    "            'chapati', 'chapstick', 'charcoal', 'charcoali', 'charcuteri', 'chardonnay',\n",
    "            'chargril', 'cheddar', 'chees', 'cheeseburg', 'cheesecak', 'cheesey', 'cheesi',\n",
    "            'cheesiest', 'cheeto', 'chemic', 'chemicali', 'cherri', 'cherry', 'cherrywood',\n",
    "            'chestnut', 'chew', 'chewabl', 'chewey', 'chewi', 'chewier', 'chick',\n",
    "            'chicken', 'chickpea', 'chilli', 'chip', 'chipboard', 'chlorin', 'chlorophyl',\n",
    "            'choc', 'chocol', 'chocolat', 'chocolatey', 'chocolati', 'chorizo', 'chowder',\n",
    "            'churro', 'chutney', 'cider', 'cideri', 'cig', 'cigar', 'cigarello',\n",
    "            'cigarett', 'cilantro', 'cinnabon', 'cinnamint', 'cinnamon', 'cinnamoni',\n",
    "            'cinnamonni', 'cinnamonpowd', 'citric', 'citrisi', 'citron', 'citronella',\n",
    "            'citrus', 'citrusi', 'clean', 'cleaner', 'cleanli', 'clear', 'clearer',\n",
    "            'clearest', 'clementin', 'clobber', 'clove', 'cloy', 'cloyingli', 'club',\n",
    "            'clump', 'clumsi', 'clumsili', 'clung', 'coal', 'coalesc', 'coalsmok', 'coca',\n",
    "            'cocoa', 'coconut', 'coconutti', 'coff', 'coffe', 'coffeebean', 'coffeeshop',\n",
    "            'cognac', 'cointreau', 'coke', 'cold', 'colder', 'coleslaw', 'complex', 'conf',\n",
    "            'confect', 'confection', 'confectionari', 'confectioneri', 'conif', 'conifer',\n",
    "            'conker', 'cool', 'cooler', 'copper', 'copperi', 'cordial', 'cork', 'corn',\n",
    "            'cornbread', 'corndog', 'cornflak', 'cornhusk', 'corni', 'cornichon',\n",
    "            'cornish', 'cornmeal', 'cotton', 'cottoni', 'cottonmouth', 'cottonwood',\n",
    "            'coughdrop', 'crackl', 'cracklin', 'cranberri', 'crap', 'crappi', 'crappier',\n",
    "            'crawfish', 'crayfish', 'crayola', 'crayon', 'cream', 'creamer', 'creami',\n",
    "            'creamier', 'creamiest', 'creamy', 'creol', 'creosot', 'crepe', 'crisco',\n",
    "            'crisp', 'crisper', 'crispi', 'crispiest', 'croissant', 'crumb', 'crumbl',\n",
    "            'crumpet', 'crunch', 'crunchi', 'cucumb', 'cumin', 'cupcak', 'curacao',\n",
    "            'currant', 'curranti', 'current', 'custard', 'custardi', 'daffodil',\n",
    "            'daiquiri', 'damp', 'dampen', 'damper', 'dandelion', 'darjeel', 'dark',\n",
    "            'darken', 'darker', 'darkest', 'darki', 'darkish', 'darkli', 'darksweet',\n",
    "            'date', 'dead', 'decay', 'deceiv', 'deceivingli', 'decompos', 'decoy', 'deep',\n",
    "            'deepen', 'deeper', 'deepest', 'degrad', 'delay', 'delight', 'delish', 'deliv',\n",
    "            'deliveri', 'demerara', 'dental', 'dentist', 'deodor', 'dessert', 'desserti',\n",
    "            'dessic', 'deterg', 'detergenti', 'devoid', 'devolv', 'dextros', 'diaper',\n",
    "            'dick', 'digest', 'digestif', 'dijon', 'diminish', 'dirt', 'dirti', 'dirtier',\n",
    "            'dirty', 'disgust', 'disgustingli', 'dissolv', 'donkey', 'dorito', 'dormant',\n",
    "            'doublemint', 'dough', 'doughi', 'doughier', 'doughnut', 'doughy',\n",
    "            'dragonfruit', 'dri', 'drier', 'driest', 'drift', 'driftwood', 'drizzl',\n",
    "            'drool', 'drumstick', 'dryer', 'dryingli', 'dryish', 'dryli', 'drymouth',\n",
    "            'dryness', 'drywal', 'dull', 'duller', 'dust', 'duster', 'dusti', 'dustier',\n",
    "            'dynam', 'dynamit', 'earth', 'earthen', 'earthi', 'earthier', 'earthy',\n",
    "            'earwax', 'eas', 'easi', 'easier', 'easiest', 'eclair', 'elderberri',\n",
    "            'element', 'elementari', 'elm', 'empti', 'endless', 'endlessli', 'energ',\n",
    "            'energet', 'energi', 'enhanc', 'enigma', 'enigmat', 'ensembl', 'entranc',\n",
    "            'entrench', 'envelop', 'ephemer', 'ephemera', 'epic', 'epicli', 'epoxi',\n",
    "            'epsom', 'espresso', 'ester', 'esterbomb', 'ethanol', 'ethanoli', 'ether',\n",
    "            'ethyl', 'eucalpytu', 'eucalypt', 'eucalyptu', 'eucalyptusi', 'evanesc',\n",
    "            'evapor', 'evapour', 'everpres', 'everywher', 'exact', 'exactli', 'excess',\n",
    "            'expresso', 'faint', 'fainter', 'faintest', 'faintli', 'fair', 'fajita',\n",
    "            'fake', 'familiar', 'fanta', 'farm', 'farmhous', 'farmi', 'farmyard',\n",
    "            'farmyardi', 'fart', 'fat', 'fatter', 'fatti', 'fattier', 'fattiest', 'feast',\n",
    "            'feather', 'featheri', 'featherweight', 'feisti', 'feistier', 'fenugreek',\n",
    "            'ferrero', 'fertil', 'field', 'fig', 'fire', 'firebomb', 'firebreath',\n",
    "            'firepit', 'fireplac', 'firewat', 'firewood', 'firm', 'firmer', 'firmli',\n",
    "            'fisherman', 'fishermen', 'fishi', 'fishiest', 'fishmarket', 'fishmong',\n",
    "            'fizz', 'fizzer', 'fizzi', 'fizzier', 'fizzl', 'flake', 'flakey', 'flaki',\n",
    "            'flamb', 'flambe', 'flamboy', 'flame', 'flammabl', 'flapjack', 'flare',\n",
    "            'flash', 'flashi', 'flat', 'flatbread', 'flatten', 'flatter', 'flattest',\n",
    "            'flavorless', 'flavourful', 'flavoursom', 'flax', 'fleet', 'flesh', 'flint',\n",
    "            'flinti', 'flor', 'flora', 'floral', 'floralhint', 'floralish', 'floss',\n",
    "            'flotsam', 'flour', 'flouri', 'flourish', 'flow', 'flower', 'floweri',\n",
    "            'flowerli', 'fluff', 'fluffi', 'fluid', 'fluidli', 'fluorid', 'flurri',\n",
    "            'flush', 'flutter', 'foam', 'foami', 'focaccia', 'fog', 'foggi', 'foliag',\n",
    "            'fondant', 'fondli', 'fondu', 'foot', 'forest', 'formaldehyd', 'fortifi',\n",
    "            'foul', 'fragran', 'fragranc', 'fragrant', 'fragrantli', 'fraich', 'frais',\n",
    "            'frappuccino', 'fresh', 'fresher', 'freshest', 'frost', 'frosti', 'froth',\n",
    "            'frothi', 'fructos', 'fructosi', 'fruit', 'fruitbread', 'fruitcak',\n",
    "            'fruitcakey', 'fruiter', 'fruiti', 'fruitier', 'fruitiest', 'fruitili', 'fudg',\n",
    "            'fudgesicl', 'fudgi', 'full', 'fuller', 'fulli', 'fungal', 'fungi', 'fungu',\n",
    "            'funk', 'funki', 'funkier', 'funkiest', 'funkish', 'funky', 'gammon', 'garlic',\n",
    "            'gasolin', 'gass', 'gassi', 'gateau', 'gatorad', 'gazpacho', 'gelati',\n",
    "            'gelatin', 'gelato', 'germin', 'gherkin', 'ghost', 'ghosti', 'ghostli', 'gin',\n",
    "            'ginger', 'gingeral', 'gingerbread', 'gingerbreadi', 'gingergread', 'gingeri',\n",
    "            'gingeriest', 'gingernut', 'gingerroot', 'gingersnap', 'ginkgo', 'ginseng',\n",
    "            'ginsengi', 'glaze', 'glucos', 'glue', 'gluey', 'gnarli', 'gooseberri',\n",
    "            'gorgonzola', 'gouda', 'grain', 'graini', 'grainier', 'grainiess', 'grainlik',\n",
    "            'grainy', 'granola', 'granul', 'granular', 'grape', 'grapefru', 'grapefruit',\n",
    "            'grapejuic', 'grapenut', 'grapey', 'grappa', 'grass', 'grassfir', 'grassi',\n",
    "            'grassier', 'grassy', 'gravel', 'gravelli', 'greengag', 'greenhous', 'gristl',\n",
    "            'gristli', 'grit', 'gritti', 'grittier', 'grizzl', 'grizzli', 'ground',\n",
    "            'groundnut', 'guacamol', 'guano', 'guava', 'gum', 'gumbal', 'gumdrop', 'gummi',\n",
    "            'gummibear', 'gummier', 'gunpowd', 'gunpowderi', 'gymsock', 'habanero',\n",
    "            'haddock', 'hairspray', 'ham', 'hammi', 'handbag', 'handsoap', 'handsoapi',\n",
    "            'hardwood', 'haribo', 'harsh', 'harsher', 'harshli', 'hawthorn', 'hay',\n",
    "            'hayish', 'haylik', 'haystack', 'hazelnut', 'hazelnutti', 'heat', 'heather',\n",
    "            'heavi', 'heavier', 'heaviest', 'heft', 'hefti', 'herb', 'herbac', 'herbaci',\n",
    "            'herbal', 'herbali', 'herbi', 'hershey', 'hickori', 'honey', 'honeycak',\n",
    "            'honeycomb', 'honeycrisp', 'honeydew', 'honeygraham', 'honeyish', 'honeymelon',\n",
    "            'honeynut', 'honeysuckl', 'honeysweet', 'hop', 'hoppi', 'hoppy', 'horlick',\n",
    "            'hors', 'horsepiss', 'horseradish', 'horsey', 'hot', 'hotdog', 'hotel',\n",
    "            'hotter', 'hottest', 'huckleberri', 'hulk', 'humidor', 'iceberg', 'icecream',\n",
    "            'innoffens', 'insipid', 'intens', 'intric', 'intricaci', 'intrigu',\n",
    "            'intriguingli', 'invigor', 'iodin', 'iodiney', 'jacket', 'jackfruit', 'jager',\n",
    "            'jagermeist', 'jalapeno', 'jam', 'jambalaya', 'jambon', 'jammi', 'jammier',\n",
    "            'jammy', 'jelli', 'jello', 'jellybean', 'jerk', 'jollyranch', 'juic', 'juicer',\n",
    "            'juicey', 'juici', 'juicier', 'juiciest', 'juicyfruit', 'kahlua', 'kebab',\n",
    "            'kerosen', 'ketchup', 'keton', 'kettlecorn', 'keylim', 'kick', 'kicker',\n",
    "            'kind', 'kinder', 'kipper', 'kipperi', 'kirsch', 'kiwi', 'kiwifruit',\n",
    "            'kiwikiwi', 'kombucha', 'koolaid', 'lacquer', 'lactic', 'lactos', 'lard',\n",
    "            'lardi', 'larg', 'larger', 'largest', 'late', 'latent', 'later', 'latest',\n",
    "            'latex', 'latexi', 'lather', 'layer', 'lazi', 'leaf', 'leafi', 'leafier',\n",
    "            'leak', 'leaki', 'lean', 'leather', 'leatherbound', 'leatheri', 'leatherish',\n",
    "            'leatherwood', 'leek', 'lemon', 'lemonad', 'lemonbread', 'lemoncello',\n",
    "            'lemongrass', 'lemoni', 'lentil', 'lettuc', 'lichen', 'lick', 'lickabl',\n",
    "            'licoric', 'licoricey', 'lifeless', 'light', 'lighter', 'lightest', 'lightish',\n",
    "            'lightweight', 'lilac', 'lime', 'limeston', 'limestoney', 'limestoni', 'limey',\n",
    "            'limon', 'limoncello', 'lindt', 'lint', 'lipstick', 'lipton', 'liqueur',\n",
    "            'liquor', 'listerin', 'liveli', 'loam', 'loami', 'lobster', 'lollipop', 'long',\n",
    "            'longer', 'longest', 'longish', 'lush', 'macadamia', 'macaroni', 'macaroon',\n",
    "            'mace', 'mackerel', 'madeira', 'malt', 'malti', 'maltier', 'maltiest',\n",
    "            'maltman', 'maltodextrin', 'malty', 'mandarin', 'mango', 'maplewood',\n",
    "            'maraschino', 'margarita', 'marigold', 'marijuana', 'marinad', 'marinara',\n",
    "            'maritim', 'maritimey', 'marjoram', 'marlboro', 'marmalad', 'marmit',\n",
    "            'marmiti', 'marsala', 'marsh', 'marshi', 'marshland', 'marshmallow',\n",
    "            'marzipan', 'mascarpon', 'massag', 'massiv', 'matchstick', 'matchsticki',\n",
    "            'mayonnais', 'mcvite', 'mead', 'meadow', 'meadowi', 'meat', 'meatbal', 'meati',\n",
    "            'meatier', 'meaty', 'medal', 'media', 'medic', 'medicin', 'medium',\n",
    "            'mediumish', 'melba', 'mellow', 'mellowest', 'melon', 'melonbread', 'melondew',\n",
    "            'meloni', 'melt', 'melti', 'menthol', 'mentholi', 'metal', 'metali', 'mezcal',\n",
    "            'milk', 'milki', 'milkshak', 'mincemeat', 'mineral', 'minerali', 'mint',\n",
    "            'minti', 'mintier', 'mintish', 'minty', 'mocha', 'moist', 'molass', 'mollusk',\n",
    "            'monster', 'monstrous', 'moonshin', 'morello', 'moscat', 'moss', 'mossi',\n",
    "            'mossier', 'mouthburn', 'mouthtingl', 'muck', 'mud', 'muddi', 'muesli',\n",
    "            'muffin', 'muggi', 'muggier', 'mulberri', 'mulch', 'mulchi', 'multigrain',\n",
    "            'muscat', 'muscatel', 'muscovado', 'muscular', 'museum', 'mush', 'mushi',\n",
    "            'mushroom', 'mushroomi', 'musk', 'muski', 'muskier', 'mustard', 'mustardi',\n",
    "            'musti', 'mustier', 'musty', 'mute', 'mutton', 'naan', 'nacho', 'nailpolish',\n",
    "            'nectar', 'nectari', 'nectarin', 'nesquik', 'new', 'newish', 'nice', 'nickel',\n",
    "            'nicotin', 'nougat', 'nougati', 'novel', 'novelti', 'numb', 'nut', 'nutella',\n",
    "            'nutmeg', 'nutmeggi', 'nutmegi', 'nutrasweet', 'nutrisweet', 'nutt', 'nutti',\n",
    "            'nuttier', 'nutty', 'oak', 'oakey', 'oaki', 'oakier', 'oakiest', 'oakwood',\n",
    "            'oaky', 'oat', 'oatcak', 'oati', 'oatmeal', 'ocean', 'oceani', 'oceansid',\n",
    "            'oil', 'oili', 'oilier', 'oiliest', 'oily', 'ointment', 'old', 'older',\n",
    "            'oldest', 'oldish', 'oolong', 'orang', 'orangejuic', 'orangey', 'orangi',\n",
    "            'orangina', 'orchard', 'orchid', 'oregano', 'outdoorsi', 'overbak',\n",
    "            'overbalanc', 'overblown', 'overcoat', 'overpow', 'overpoweringli',\n",
    "            'overshadow', 'overwhelm', 'overwhelmingli', 'oyster', 'ozon', 'ozoney',\n",
    "            'paint', 'pancak', 'pancetta', 'panetton', 'papaya', 'paprika', 'paraffin',\n",
    "            'parafin', 'parmesan', 'parsley', 'parsnip', 'passionfruit', 'pasta', 'pastel',\n",
    "            'pasti', 'pastrami', 'pastri', 'pate', 'patisseri', 'pavlova', 'pe', 'peach',\n",
    "            'peachi', 'peanut', 'peanuti', 'pear', 'peardrop', 'peat', 'peatbomb',\n",
    "            'peated', 'peathead', 'peati', 'peatier', 'peatiest', 'peatish', 'peatreek',\n",
    "            'peatsmok', 'peaty', 'pecan', 'pecanish', 'pedestrian', 'peel', 'pepp',\n",
    "            'pepper', 'peppercorn', 'pepperi', 'pepperish', 'peppermint', 'pepperminti',\n",
    "            'pepperoni', 'peppery', 'peppi', 'peppier', 'pepsi', 'pepto', 'perfum',\n",
    "            'pernod', 'peroxid', 'pesto', 'petalpetrol', 'petroleum', 'pewter', 'pharmaci',\n",
    "            'pheasant', 'phenol', 'philadelphia', 'pickl', 'pickli', 'pie', 'piecrust',\n",
    "            'pig', 'pimento', 'pina', 'pineappl', 'pineappley', 'pinewood', 'pinot',\n",
    "            'pinotag', 'pipe', 'pipetobacco', 'pistachio', 'pizza', 'plantain', 'planti',\n",
    "            'plastic', 'plastici', 'playdoh', 'playdough', 'pleasant', 'plum', 'plump',\n",
    "            'plywood', 'pommegran', 'pool', 'poop', 'poppadum', 'popsicl', 'poptart',\n",
    "            'porridg', 'porridgey', 'portobello', 'portwood', 'potato', 'potpourri',\n",
    "            'potroast', 'poultri', 'poundcak', 'powder', 'powderi', 'power', 'powerful',\n",
    "            'powerhous', 'prawn', 'pregnant', 'profiterol', 'prosciutto', 'prune',\n",
    "            'pruney', 'pruni', 'pulpi', 'pumpkin', 'punch', 'punchi', 'punchier',\n",
    "            'pungent', 'putrid', 'quirki', 'radish', 'rainfal', 'rainforest', 'raini',\n",
    "            'rainier', 'rainin', 'rainstorm', 'rainwash', 'rainwat', 'raisin', 'raisiney',\n",
    "            'raisini', 'ramen', 'rancid', 'rapid', 'rasp', 'raspberri', 'raspi', 'raw',\n",
    "            'rawer', 'recur', 'redwood', 'reek', 'refresh', 'reincarn', 'reinforc',\n",
    "            'reinvigor', 'relentless', 'reliabl', 'relief', 'relish', 'resin', 'resini',\n",
    "            'rhubarb', 'ribena', 'ricard', 'rice', 'rich', 'richer', 'richest', 'richli',\n",
    "            'ricotta', 'rioja', 'ripe', 'ripen', 'riper', 'risotto', 'roach', 'roast',\n",
    "            'rocket', 'rockmelon', 'romain', 'root', 'rootbeer', 'roquefort', 'rosewood',\n",
    "            'rosi', 'rosin', 'roti', 'rotisseri', 'rotten', 'rough', 'roughen', 'rougher',\n",
    "            'roughli', 'round', 'rounded', 'rounder', 'roundish', 'rubber', 'rubberi',\n",
    "            'rum', 'rumi', 'rumsoak', 'rush', 'rye', 'ryemint', 'ryespic', 'saccharin',\n",
    "            'saccharinni', 'sachet', 'saffron', 'sage', 'sake', 'salami', 'salmon',\n",
    "            'salmoni', 'salsa', 'salt', 'saltdough', 'salti', 'saltier', 'saltiest',\n",
    "            'salty', 'sambuca', 'sandalwood', 'sandlewood', 'sandlewoodyish', 'sangria',\n",
    "            'sapwood', 'sauerkraut', 'sauerkrauti', 'sauna', 'sausag', 'sausagey', 'saute',\n",
    "            'sauvignon', 'savor', 'savori', 'savorli', 'savory', 'savour', 'savouri',\n",
    "            'sawdust', 'sawdusti', 'sawgrass', 'sawmil', 'sawn', 'scallop', 'schnitzel',\n",
    "            'scone', 'scorch', 'scorcher', 'scorchingli', 'scotchi', 'scotchier', 'sea',\n",
    "            'seabass', 'seafoam', 'seafood', 'seafoodi', 'seafoodish', 'seasalt',\n",
    "            'seasmok', 'seaspray', 'seassid', 'seawe', 'seaweedi', 'sediment', 'seed',\n",
    "            'seedi', 'seltzer', 'semisweet', 'semolina', 'sensibl', 'sequoia', 'shale',\n",
    "            'shallow', 'shampoo', 'sharp', 'sharpe', 'sharpen', 'sharper', 'sharpi',\n",
    "            'sharpish', 'shellfish', 'sherbert', 'sherberti', 'sherbet', 'sherbeti',\n",
    "            'sherri', 'sherriest', 'sherrri', 'sherry', 'sherrybomb', 'sherryfruit',\n",
    "            'sherryish', 'sherryland', 'sherrylici', 'sherrynot', 'sherrywood', 'ship',\n",
    "            'shipwreck', 'shiraz', 'shisha', 'shoe', 'shoepolish', 'short', 'shortbread',\n",
    "            'shortcrust', 'shortest', 'shortish', 'shrimp', 'shroom', 'shroomi', 'shrub',\n",
    "            'sickli', 'silk', 'silki', 'silkier', 'simpl', 'sippabl', 'sirloin', 'sizzl',\n",
    "            'skunk', 'skunki', 'slim', 'slimi', 'slipperi', 'sloe', 'sloeberri', 'sloppi',\n",
    "            'slouch', 'slow', 'sludg', 'smog', 'smoke', 'smokei', 'smokepeat', 'smoker',\n",
    "            'smokey', 'smoki', 'smokier', 'smokiest', 'smokish', 'smolder', 'smolderi',\n",
    "            'smooth', 'smoothest', 'smoothi', 'smoothish', 'sneaki', 'soft', 'softest',\n",
    "            'solid', 'soot', 'sooti', 'sophist', 'sorbet', 'souchong', 'souffl', 'soup',\n",
    "            'soupi', 'sour', 'sourdough', 'sourdoughi', 'soya', 'soybean', 'soymilk',\n",
    "            'spearmint', 'spearminti', 'spic', 'spice', 'spicebomb', 'spicebox', 'spicei',\n",
    "            'spicey', 'spici', 'spiciest', 'spicy', 'spikey', 'spiki', 'spiky', 'spinach',\n",
    "            'spongi', 'stableyard', 'stale', 'starburst', 'starfruit', 'steak',\n",
    "            'steakhous', 'steril', 'stew', 'stink', 'stinker', 'stinki', 'strawberri',\n",
    "            'stretch', 'sugar', 'sugarcan', 'sugari', 'sugarloaf', 'sugarplum', 'sugary',\n",
    "            'sulfur', 'sulfuri', 'sulphur', 'sulphuri', 'sultan', 'sultana', 'sultri',\n",
    "            'sumatra', 'sumatran', 'suncream', 'sunkist', 'superglu', 'swamp', 'swampi',\n",
    "            'sweet', 'sweetbread', 'sweetcorn', 'sweetest', 'sweeti', 'sweetshop', 'swell',\n",
    "            'syrup', 'syrupi', 'tabasco', 'taffi', 'tahini', 'talc', 'tamarind', 'tan',\n",
    "            'tandoori', 'tang', 'tangerin', 'tangibl', 'tango', 'tanic', 'tanni', 'tannic',\n",
    "            'tannin', 'tannini', 'taper', 'tapioca', 'tar', 'taragon', 'tarmac',\n",
    "            'tarragon', 'tarri', 'tart', 'tarter', 'tarti', 'tartish', 'tequila',\n",
    "            'teriyaki', 'thick', 'thickest', 'thin', 'thinner', 'thinnest', 'thinnish',\n",
    "            'thyme', 'tikka', 'tingli', 'tingly', 'tiramisu', 'tire', 'tobacco',\n",
    "            'tobaccoish', 'tobleron', 'toffe', 'toffee', 'tortilla', 'trail', 'trailmix',\n",
    "            'tranquil', 'treacl', 'treacley', 'treacli', 'tree', 'treefruit', 'treenut',\n",
    "            'tropicana', 'tropifrutti', 'truffl', 'trunk', 'tubbi', 'tulip', 'tuna',\n",
    "            'turkey', 'turnip', 'turpentin', 'turtlegrass', 'twig', 'twiggi', 'twigi',\n",
    "            'twinki', 'twinkl', 'twizzler', 'tyre', 'umami', 'unapologet', 'unassum',\n",
    "            'unbalanc', 'unctuous', 'underag', 'underbak', 'underbit', 'undercook',\n",
    "            'undergird', 'underproof', 'understat', 'underwhelm', 'undevelop', 'undramat',\n",
    "            'uneven', 'unexcit', 'unimpeach', 'unimpress', 'uninspir', 'unravel',\n",
    "            'unspectacular', 'uplift', 'vagu', 'vaguest', 'valpolicella', 'vanila',\n",
    "            'vanilin', 'vanilla', 'vanillaroot', 'vanilli', 'vanillia', 'vanillin',\n",
    "            'vanish', 'vannila', 'vapor', 'vapour', 'varnish', 'varnishi', 'vaselin',\n",
    "            'veg', 'vegemit', 'veget', 'veggi', 'vegit', 'vegtabl', 'velvet', 'velveti',\n",
    "            'velvetti', 'veneer', 'venison', 'vibrant', 'vigor', 'vigour', 'vinaigrett',\n",
    "            'vindaloo', 'vinegar', 'viney', 'vineyard', 'violent', 'violet', 'viscous',\n",
    "            'vodka', 'voluptu', 'vomit', 'vulgar', 'wafer', 'waffl', 'walnut', 'walnutti',\n",
    "            'wasabi', 'wassail', 'watercress', 'watermelon', 'watermeloni', 'wax', 'waxi',\n",
    "            'waxiest', 'waxless', 'waxy', 'weetabix', 'weighti', 'weird', 'weirdest',\n",
    "            'welli', 'wellington', 'wetsuit', 'whallop', 'wham', 'whammi', 'whamo',\n",
    "            'wheat', 'wheatgrass', 'wheati', 'wheatmeal', 'whimsic', 'whirlwind',\n",
    "            'whitefish', 'wild', 'wildberri', 'windswept', 'wine', 'winegum', 'winey',\n",
    "            'wintermelon', 'wintermint', 'wispi', 'wood', 'woodburn', 'woodchar',\n",
    "            'woodchip', 'wooden', 'woodfir', 'woodglu', 'woodi', 'woodish', 'woodland',\n",
    "            'woodpil', 'woodsap', 'woodsmok', 'woodsmokey', 'woodspic', 'woody', 'wool',\n",
    "            'woolen', 'wooli', 'wormwood', 'yeast', 'yeasti', 'yoghurt', 'yoghurti',\n",
    "            'yogurt', 'yogurti', 'young', 'youngest', 'youngish', 'yum', 'yumm', 'yummi',\n",
    "            'zest', 'zesti', 'zesty', 'zing', 'zinger', 'zingi', 'zombi']\n",
    "len(wordlist)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9964ddc2",
   "metadata": {},
   "source": [
    "OK, so now I have my overall set of stems that I think are whisky-tasting-related. I got it down from 15000-odd, so it's not too bad. Something I could do here to improve things would be to use a dictionary and start combining some of these stems that are similar in meaning. Another thing I could do is explore different n-gram lengths. I'm using singular n-grams here, but clearly that's reductionistic - e.g. \"peat monster\" means something distinct to \"peat\" + \"monster\" (you can also have sherry monsters). \n",
    "\n",
    "Since I have the tools to set up a vectoriser now, and I have the df_collated dataframe, I consider the data cleaning to be complete. The next notebook will be the construction and tuning of a ML classifier algorithm for predicting the brand of whisky from the nose, palate, and finish notes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "3ae67d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_collated.to_csv('./data/branded.csv')\n",
    "\n",
    "with open('./data/tasting_notes.txt', 'w') as file:\n",
    "    file.writelines([wordlist[i] + '\\n' for i in range(len(wordlist))])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eb339b6",
   "metadata": {},
   "source": [
    "Actually... not quite done. Given the rather meagre performance of my classifier algorithm, I decided I'd also try a more selective predictor by restricting things to the domain of single malt scotch whisky to see if it resulted in a better performance. So what follows is a similar bit of code to create a collated list of singlemalts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f2fae345",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5633 branded successfully. 246 failed. Time taken: 14 seconds.\n",
      "5679 branded successfully. 159 failed. Time taken: 4 seconds.\n",
      "3864 branded successfully. 245 failed. Time taken: 8 seconds.\n",
      "2103 branded successfully. 23 failed. Time taken: 1 seconds.\n",
      "1151 branded successfully. 6 failed. Time taken: 0 seconds.\n",
      "438 branded successfully. 158 failed. Time taken: 0 seconds.\n"
     ]
    }
   ],
   "source": [
    "# Empty dataframe to store correctly-branded rows and incorrectly-branded rows\n",
    "df_collated = pd.DataFrame(columns = ['name', 'rating', 'region', 'nose', 'palate', 'finish', 'brand'])\n",
    "df_fails = pd.DataFrame(columns = ['name', 'rating', 'region', 'nose', 'palate', 'finish', 'brand'])\n",
    "\n",
    "# List containing tuples of the names of the dataframes for each single malt scothch region \n",
    "# and the corresponding list for the whisky names in that region\n",
    "single_malt_scotch_df_and_lists_tuples = [(df_speyside, speyside_list),\n",
    "                                          (df_islay, islay_list),\n",
    "                                          (df_highland, highland_list),\n",
    "                                          (df_islands, islands_list),\n",
    "                                          (df_campbeltown, campbeltown_list),\n",
    "                                          (df_lowland, lowland_list)]\n",
    "\n",
    "# Iterate through SM scotch regions:\n",
    "for region_tuple in single_malt_scotch_df_and_lists_tuples:\n",
    "    # counters\n",
    "    start_time = time.time()\n",
    "    collatedrows = df_collated.shape[0]\n",
    "    failsrows = df_fails.shape[0]\n",
    "    \n",
    "    # Apply function\n",
    "    df_collated, df_fails = get_brand_names_per_region(region_tuple, df_collated, df_fails)\n",
    "    \n",
    "    print('{} branded successfully. {} failed. '.format(\n",
    "        df_collated.shape[0] - collatedrows, df_fails.shape[0] - failsrows), end=\"\")\n",
    "    print('Time taken: {} seconds.'.format(round(time.time() - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "1f37f13b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_collated.to_csv('./data/branded_singlemalts.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}