Match_to_EW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "319c907b",
   "metadata": {},
   "source": [
    "## Analysis set-up"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35140caa",
   "metadata": {},
   "source": [
    "### Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f4d3d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "#import seaborn as sns\n",
    "from numpy import median\n",
    "import geopandas as gpd\n",
    "#import contextily as cx\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import string\n",
    "import random\n",
    "import numpy as np\n",
    "import re\n",
    "pd.set_option('display.max_rows', 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66969466",
   "metadata": {},
   "source": [
    "### Set file path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e52349ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = '/Users/franpontin/OneDrive - University of Leeds/CDRC/Projects/IGD/Analysis/product_data/'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b38748d2",
   "metadata": {},
   "source": [
    "### Define functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df29cf51",
   "metadata": {},
   "source": [
    "#### Function to report classification progress "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bddb46e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress(df):\n",
    "  #gives progress on EW calssificaiton\n",
    "   print('N left to classify:',len(df.loc[df['Eatwell_Segment']=='Not Food',:]),\n",
    "      '(Percentage classified =', \n",
    "      np.round(100-len(df.loc[df['Eatwell_Segment']=='Not Food',:])/len(df)*100,2), '%)')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af479748",
   "metadata": {},
   "source": [
    "#### Function to match product descritpiton/name to Eatwell categroy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83a432e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def match_to_EW(df, list_of_columns):\n",
    "    df[list_of_columns] =df[list_of_columns].astype('string')\n",
    "    n=len(list_of_columns)\n",
    "    ewList =['EW_' + x for x in list(map(str, list(range(0, n))))]\n",
    "    df[ewList]=np.nan\n",
    "    for i in range(len(ew_cat_wordLists)):\n",
    "        wordList = ew_cat_wordLists[i]\n",
    "        print(i+1, 'categories matched of', len(ew_cat_wordLists)) \n",
    "        for word in wordList:\n",
    "            for j in range(len(w_segList)):\n",
    "            # find where there is a word to macth, \n",
    "            # and it has not yet been matched in another cat (order of cat names important)\n",
    "            # match the word to the word list and assign EW cat to word\n",
    "                        df.loc[(df[w_segList[j]].notnull())&\n",
    "                        (~df[ewList[j]].notnull())&\n",
    "                        (df[w_segList[j]].str.contains(word, case=False, )),#flags=re.IGNORECASE, regex=True\n",
    "                        ewList[j]] = ew_cat[i]\n",
    "    df['nunique_EW']=df[ewList].apply(pd.Series.nunique, axis=1)\n",
    "    #example['EW_multi_cats'] =example[['EW_0','EW_1','EW_2','EW_3','EW_4','EW_5','EW_6','EW_7','EW_8','EW_9','EW_10','EW_11','EW_12']].fillna(\"\").astype(str).fillna(\"\").apply(lambda x: ' '.join(x), axis=1).fillna(\"\")\n",
    "    df['EW_multi_cats'] =df[ewList].apply(lambda x: ', '.join(x[x.notnull()]), axis = 1)\n",
    "    df['total_seg_matched']=df[ewList].count(axis=1)  \n",
    "    mode_n =df[['EW_0','EW_1','EW_2','EW_3','EW_4','EW_5','EW_6']].mode(axis=1).shape[1]\n",
    "    modeList =['mode_' + x for x in list(map(str, list(range(1, mode_n+1))))]\n",
    "    df[modeList] =df[ewList].mode(axis=1)\n",
    "    df['Eatwell_Segment'] = 'Not Food'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff612ee1",
   "metadata": {},
   "source": [
    "#### Function to assign Eatwell classification based on matching, including detail of logic behind the match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f99fd4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def assign_EW(df, conditions, confidence_val, confidence_scale, match_detail, ew_cat):\n",
    "    print('N='+str(len(df.loc[conditions,:])))\n",
    "    df.loc[conditions, 'class_confidence_calc']=confidence_val\n",
    "    df.loc[conditions, 'class_confidence']= confidence_scale\n",
    "    df.loc[conditions, 'match_detail']= match_detail\n",
    "    df.loc[conditions&(df['Eatwell_Segment']=='Not Food'),'Eatwell_Segment'] = ew_cat\n",
    "    print('Match conditions:',match_detail)\n",
    "    progress(df)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50f9d4b3",
   "metadata": {},
   "source": [
    "### Read in test product data and additional list of fruit and veg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a54a5afb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(file_path+'product_data.csv')\n",
    "veg_list =pd.read_csv(file_path+'veg_list.csv',index_col=0)\n",
    "\n",
    "veg_list =list(veg_list['name'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b6be1ee",
   "metadata": {},
   "source": [
    "## Define products in each Eatwell category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bf96ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "ew_fnv = veg_list+['courgette','dudhi','mooli', 'gourd','melon','currants', 'sultanas','swede','veg',\n",
    "                   'fruit','gherkin','pickle','olives']\n",
    "ew_car = ['bread','cereal','flour','pasta','pastry','potato','rolls',r'\\brice\\b',\n",
    "          'waffles','zwieback', 'muffin', 'teff', 'bagel', r'\\bcorn\\b', 'spaghetti',\n",
    "          'whole wheat', 'crackers', 'noodles', 'wheat', 'couscous','cous cous', 'tortilla',\n",
    "          'nann','taco','pitta','wrap','chapati','chappatti', 'chapatti','sourdough',r'\\bpave\\b','boule','bloomer','paratha',\n",
    "          'bao','cassava','naan','porridge','museli', 'muesli','baguette','quinoa','ciabatta',r'\\broti\\b',\n",
    "          'sourdough', 'baps', 'brioche','pennoni', 'loaf','fusilli','macaroni','focaccia'] # zwieback is carb or disc? doesn't matter for this tho\n",
    "\n",
    "ew_ptn = ['beef','chicken','domestic eggs', 'lamb','fish','frankfurter',r'\\bham\\b',\n",
    "          'liver loaf','meat','nut','pork','sausage','turkey', 'crustaceans',\n",
    "          'bean', 'chickpea', 'veal','poultry', 'steak', 'neck', 'mince',\n",
    "         'fillet', 'loin', 'ostrich', 'egg', 'mollusk', 'emu', 'goose', 'duck',\n",
    "          'Goat', 'lentils', 'marrow', 'flesh', 'tofu','beancurd', 'prawn', 'hummus', 'houmous',\n",
    "         'haddock','salmon',r'\\bcod\\b','capers','tuna','breast','lobster','thighs','drumstick',\n",
    "          'prosciutto','wagyu','joint', 'quorn', 'bacon', 'brisket','burger','almonds', \n",
    "          r'\\bnuts\\b','pistachios','pecans','butter beans','rotisserie','mortadella','kala chana',\n",
    "         'liver','pig in blanket','seafood'] # marrow very fatty\n",
    "\n",
    "ew_oil = ['butter','margarine',r'\\boil\\b', 'lard','spreads','vegetable oil', 'sunfower oil']\n",
    "ew_dai = ['milk','whole milk','butter milk','cream','cheese','curd','yogurt', 'yoghurt','shortening',\n",
    "          r'\\bmilk alternatives\\b', r'\\blactose free drinks\\b', r'\\boat & nut drinks\\b','kefir','cheesy',\n",
    "          'brie','mozzerella','mozzarella','cheesy single', 'cheddar', 'gloucester','leicester','soya','halloumi',\n",
    "          'wensleydale','coconut milk','stilton', 'paneer','feta','Lactofree','fraiche']\n",
    "\n",
    "ew_dis = ['candy','cake','sweetener','chocolate','choc','dessert','ice cream',\n",
    "          'mustard','mayonnaise','salty snack','pudding powder','popcorn',\n",
    "          'sweet spreads','syrup','sugar','jam','ketchup', 'fudge', 'cocoa',\n",
    "          'snack products','sauce', 'cookie','pastries', 'pudding', 'pancake',\n",
    "         'candies', 'wafer', 'fries', 'crumpets', 'muffins', 'crisps','jam','condiments',\n",
    "          'gravy', 'yeast', 'marmalade', 'conserve','caramel','brownie','bakewell','profiterole',\n",
    "          'Soreen','chips','biscuit', 'scone', 'snack', 'marzipan','toffee','custard',r'\\bicing\\b',\n",
    "          'jelly','stollen','poppadom', 'poppadum','tiramisu','croissant','macaroon','mringue','cheesecake','salad cream',\n",
    "         r'\\bpain au rasin\\b','shortbread','honey','salami','cheesecake','panna cotta','sweets', 'mints', 'donner kerbab',\n",
    "          'hash brown',r'\\bhot cross bun\\b','pick and mix','sharing bag','crouton', 'hot cross bun','bakery',\n",
    "          'dressing','flapjack','swiss roll','pain au raisin','mallow','preserve','trifle']\n",
    "\n",
    "ew_alc = ['brandy','beer','wine','prosecco','liqueur','liquor',r'\\brum\\b','whisky', 'cocktail',\n",
    "          'Liqueur', r'\\bgin\\b','sauvignon','ABV','%','daiquiry','martini','cava','rioja','pilsner','sherry','cider',r'\\bport\\b']\n",
    "ew_com = ['nut/prunes','soup','Instant food products',r'\\bfrozen meal\\b', 'pizza', 'lasagna',\n",
    "          'lasagne','pie', 'burito','pakora','sandwich', 'ready meal', r'\\noodle pots\\b',\n",
    "          'haggis','takeaway', 'meal', 'curry','balti','masala','quiche',r'\\bpate\\b','bolognese',\n",
    "          'bolognaise','carbonara','platter','gyoza', 'coleslaw','pasta bake','granola','sping roll',\n",
    "          'scampi','croquette','sandwich','party food','chilli con carne','hot dog','kmoussaka','enchilada',\n",
    "          'fajita',r'\\bstir fry\\b', 'katsu','salsa']\n",
    "\n",
    "# Add cateogry for infant formula etc. (not incl;uded in adult recommendations)\n",
    "ew_inf = ['formula','babyfood', 'toddler','infant','formula']\n",
    "\n",
    "ew_other =['seasoning', 'baking', 'spices','medical','coriander','HERBS & SPICES',  r'\\bice cube\\b', 'salt']\n",
    "# add beverage cat.\n",
    "\n",
    "ew_bev = ['juice', 'beverage', 'fizzy drink', 'cola', r'\\bcoke\\b',r'\\bminneral water\\b', 'mineral','drink',\n",
    "          'coffee', 'sparkling water','tea', 'lucozade','decaf','tonic','soda','Kombucha', 'bottle', r'\\bcan\\b',\n",
    "          'Sarsaparilla','smoothie','latte', r'\\bginger ale\\b', 'water', r'\\balcohol free\\b', 'ginger ale']\n",
    "\n",
    "# add exclude - food items without significant nutritional contribution in terms of eat well\n",
    "# e.g. spices, herbs, make sure to check salt and sugar)\n",
    "\n",
    "#ew_exc = []\n",
    "\n",
    "ew_cat = ['Fruit & Veg', 'Starchy Carbs', 'Protein', 'Oils & Spreads', 'Dairy & Alt.',\n",
    "          'Discretionary', 'Alcoholic Bev.','Composite', 'Infant', 'Non Alcoholic Bev.', 'Other']\n",
    "\n",
    "ew_cat_wordLists = [ew_fnv,ew_car,ew_ptn,ew_oil,ew_dai,ew_dis,ew_alc,ew_com, ew_inf, ew_bev, ew_other]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "311a50cb",
   "metadata": {},
   "source": [
    "### Match each product name/description to Eatwell category/catergories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "536494f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "w_segList =['Shelf_1','Shelf_2', 'Shelf_3', 'Shelf_4', 'Shelf_5','MDSE Subcategory Name','MDSE Category Name']\n",
    "match_to_EW(df, w_segList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff4c9552",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Percentage with no match on inital join: '+str(round(len(df.loc[df['total_seg_matched']<1])/len(df)*100,2))+\"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89e84060",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=df, \n",
    "          conditions=((df['nunique_EW']==1 )& (df['total_seg_matched']>1)), \n",
    "          confidence_val=1.00, \n",
    "          confidence_scale='High',\n",
    "          match_detail='Product name matched to single EW category more than once',\n",
    "          ew_cat=df['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa58dfd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=df, \n",
    "          conditions=((df['nunique_EW']==1 )& (df['total_seg_matched']==1)), \n",
    "          confidence_val=1.00, \n",
    "          confidence_scale='Fairly High',\n",
    "          match_detail='Product name matched to single EW category once',\n",
    "          ew_cat=df['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd751ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=df, \n",
    "          conditions=((df['nunique_EW']>=2)&\n",
    "            (df['mode_2'].isna())&\n",
    "            (df['mode_3'].isna())&\n",
    "            (df['Eatwell_Segment']=='Not Food')),\n",
    "          confidence_val=df['nunique_EW']/df['total_seg_matched'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail='Product name matched to multiple EW categories, most common and first cateogry matched are the same',\n",
    "          ew_cat=df['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed339124",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['match_detail'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa19c203",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=df, \n",
    "          conditions=((df['total_seg_matched']==2)&(~df['mode_2'].isna())&(df['Eatwell_Segment']=='Not Food')&\n",
    "                          # and one of those EW cats is composite class as composite\n",
    "                          (df['EW_multi_cats'].str.contains('Composite', case=False))),\n",
    "          confidence_val=1/df['nunique_EW'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail=\"Product name matched to multiple EW categories, more than one most common EW segment (equal likelihood), however one of the most common EW segments is a `Composite` food so classed as such \",\n",
    "          ew_cat='Composite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5370954",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=df, \n",
    "          conditions=(df['total_seg_matched']==2)&(~df['mode_2'].isna())&(df['Eatwell_Segment']=='Not Food')&\n",
    "       # if one of the EW cats mathced is 'Discretioanry' then class as discreitoanry\n",
    "       (df['EW_multi_cats'].str.contains('Discretionary', case=False)),\n",
    "          confidence_val=1/df['nunique_EW'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail=\"Product name matched to multiple EW categories, more than one most common EW segment (equal likelihood), however one of the most common EW segments is a `Disceretionary` food so classed as such \",\n",
    "          ew_cat='Discretionary')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83205355",
   "metadata": {},
   "source": [
    "### For unmatched products split the product description and classify each component of the product description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fe5c674",
   "metadata": {},
   "outputs": [],
   "source": [
    "print ('For the remaining', df.loc[(df['Eatwell_Segment']=='Not Food'),:].shape[0], 'products split the product descirption and match based on indvidual components of the description')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7902c3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "split_name =df.loc[(df['Eatwell_Segment']=='Not Food'),'name'].str.split(' ', expand=True)\n",
    "split_name.columns =['w0','w1','w2','w3','w4','w5','w6','w7','w8','w9','w10','w11',\n",
    "                     'w12','w13','w14','w15']\n",
    "split_name =split_name.merge(df[['name','Retailer Barcode']], left_index=True, right_index=True, how='left')\n",
    "w_segList =['w0','w1','w2','w3','w4','w5','w6','w7','w8','w9','w10','w11',\n",
    "                     'w12','w13','w14','w15']\n",
    "\n",
    "match_to_EW(split_name, w_segList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81705ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=split_name, \n",
    "          conditions=((split_name['nunique_EW']==1 )& (split_name['total_seg_matched']>1)), \n",
    "          confidence_val=1.00, \n",
    "          confidence_scale='High',\n",
    "          match_detail='Product name matched to single EW category more than once',\n",
    "          ew_cat=df['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "948d28e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=split_name, \n",
    "          conditions=((split_name['nunique_EW']==1 )& (split_name['total_seg_matched']==1)), \n",
    "          confidence_val=1.00, \n",
    "          confidence_scale='Fairly High',\n",
    "          match_detail='Product name matched to single EW category once',\n",
    "          ew_cat=df['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "348c94be",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=split_name, \n",
    "          conditions=((split_name['nunique_EW']>=2)&\n",
    "            (split_name['mode_2'].isna())&\n",
    "            (split_name['mode_3'].isna())&\n",
    "            (split_name['Eatwell_Segment']=='Not Food')),\n",
    "          confidence_val=split_name['nunique_EW']/split_name['total_seg_matched'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail='Product name matched to multiple EW categories, most common and first cateogry matched are the same',\n",
    "          ew_cat=split_name['mode_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d0692da",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=split_name, \n",
    "          conditions=((split_name['total_seg_matched']==2)&(~split_name['mode_2'].isna())&(df['Eatwell_Segment']=='Not Food')&\n",
    "                          # and one of those EW cats is composite class as composite\n",
    "                          (split_name['EW_multi_cats'].str.contains('Composite', case=False))),\n",
    "          confidence_val=1/split_name['nunique_EW'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail=\"Product name matched to multiple EW categories, more than one most common EW segment (equal likelihood), however one of the most common EW segments is a `Composite` food so classed as such \",\n",
    "          ew_cat='Composite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a76ee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=split_name, \n",
    "          conditions=(split_name['total_seg_matched']==2)&(~split_name['mode_2'].isna())&(split_name['Eatwell_Segment']=='Not Food')&\n",
    "       # if one of the EW cats mathced is 'Discretioanry' then class as discreitoanry\n",
    "       (split_name['EW_multi_cats'].str.contains('Discretionary', case=False)),\n",
    "          confidence_val=1/split_name['nunique_EW'], \n",
    "          confidence_scale='Good',\n",
    "          match_detail=\"Product name matched to multiple EW categories, more than one most common EW segment (equal likelihood), however one of the most common EW segments is a `Disceretionary` food so classed as such \",\n",
    "          ew_cat='Discretionary')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96e1afe1",
   "metadata": {},
   "source": [
    "### Combine product description matched data with disaggregated matches "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "102ae822",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df =df\n",
    "new_df = new_df.drop(split_name.index)\n",
    "new_df =pd.concat([new_df, split_name], axis=0)\n",
    "new_df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0970aa38",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=new_df, \n",
    "          conditions=((new_df['name'].str.contains('flour', case=False))&(new_df['Eatwell_Segment']=='Not Food')),\n",
    "          confidence_val= 1.00,\n",
    "          confidence_scale='Fairly High',\n",
    "          match_detail=\"Product name contains flour so classed as starchy carb.\",\n",
    "          ew_cat =  'Starchy Carbs')\n",
    "          \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&(new_df['nunique_EW']==2) \n",
    "               &(new_df['EW_multi_cats']=='Fruit & Veg, Non Alcoholic Bev.')\n",
    "               &(new_df['name'].str.contains('in '))), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='ok', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, fruit & veg and non alcoholic bev, if contains `in` class as fruit and veg else class as non-alocholic bev.', \n",
    "          ew_cat='Fruit & Veg')\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&(new_df['nunique_EW']==2) \n",
    "               &(new_df['EW_multi_cats']=='Fruit & Veg, Non Alcoholic Bev.')\n",
    "               &(~new_df['name'].str.contains('in '))), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='ok', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, fruit & veg and non alcoholic bev, if contains `in` class as fruit and veg else class as non-alocholic bev.', \n",
    "          ew_cat='Non Alcoholic Bev.')            \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&\n",
    "                                  (new_df['nunique_EW']==2)&\n",
    "                                  (new_df['EW_multi_cats']=='Fruit & Veg, Starchy Carbs')), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='ok', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, protein and carb. or protein and fruit/veg. clasify as composite', \n",
    "          ew_cat='Starchy Carbs')    \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&\n",
    "                                  (new_df['nunique_EW']==2)&\n",
    "                                  (new_df['EW_multi_cats']=='Protein, Starchy Carbs')), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='ok', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, protein and carb. or protein and fruit/veg. clasify as composite', \n",
    "          ew_cat='Starchy Carbs')    \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')& \n",
    "               (new_df['name'].str.contains('%'))), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='Good', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, if product name contains % (alcohol) class as alcoholic bev.', \n",
    "          ew_cat='Alcoholic Bev.')   \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&\n",
    "                                  (new_df['nunique_EW']==2)&\n",
    "                                  (new_df['name'].str.contains('yogurt|yoghurt',case=False))), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='Good', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, if product name contains yoghurt class as dairy', \n",
    "          ew_cat='Dairy & Alt.')  \n",
    "\n",
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&(new_df['name'].str.contains('with'))), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='Good', \n",
    "          match_detail= 'Product name matched to 2 EW categorys, if the name contians `with` classify as composite', \n",
    "          ew_cat='Dairy & Alt.')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b2e789b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&(new_df['nunique_EW']>=2)), \n",
    "          confidence_val=0.5, \n",
    "          confidence_scale='Good', \n",
    "          match_detail= 'Product name matched to 2 or more EW categorys, if not already matched classify as composite', \n",
    "          ew_cat='Composite')   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20af5d14",
   "metadata": {},
   "outputs": [],
   "source": [
    "assign_EW(df=new_df,\n",
    "          conditions=((new_df['Eatwell_Segment']=='Not Food')&(new_df['name'].str.contains(r'\\bice\\b', case=False))), \n",
    "          confidence_val=0.9, \n",
    "          confidence_scale='Good', \n",
    "          match_detail= 'Product name contians `ice` classify as other', \n",
    "          ew_cat='Other')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adc69c14",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "new_df['Eatwell_Segment'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6856f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "list(new_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f12c9fac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use interactive histograms to identify outliers\n",
    "fig = px.histogram(new_df.fillna(''), x='kcal/100g', y='Protein/100g', color=\"Eatwell_Segment\", marginal=\"rug\",\n",
    "                   hover_data=['Product Name','Retailer Product Name']) #hover_data selects the variable which appears when hovering over the data points\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b95a9b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use interactive histograms to identify outliers\n",
    "fig = px.histogram(new_df.fillna(''), x='kcal/100g', y='Sugar/100g', color=\"Eatwell_Segment\", marginal=\"rug\",\n",
    "                   hover_data=['Product Name','Retailer Product Name']) #hover_data selects the variable which appears when hovering over the data points\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01b0384a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}