Text-side/IngredientsEmbedding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "IngredientsEmbedding.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "LfwkzWRcoUr-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0c6150fd-ec46-4d66-f957-787ae87a3a85"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SN0e-BazRY9I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path = '/content/drive/My Drive/new_ingredients.txt'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJgGQFRkRdG_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(path, \"r\") as file1:\n",
        "    FileasList = file1.readlines()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uBdbP3PQRjBA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import ast "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FmXMRAgMSMKf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKJd5HCaSQ2d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for obj in FileasList:\n",
        "  res = ast.literal_eval(obj)\n",
        "  data.append(res)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O7wM_k7CThXT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8b68f254-c98d-40c2-c4c5-8a2a80e078e1"
      },
      "source": [
        "len(data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1029719"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6rxwMnXkWuc7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import print_function\n",
        "import pandas as pd\n",
        "# for plotting\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import numpy as np\n",
        "pd.pandas.set_option('display.max_columns', None)\n",
        "#pip install --upgrade gensim\n",
        "import json\n",
        "import operator\n",
        "import collections\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import nltk\n",
        "from gensim.models import word2vec\n",
        "from sklearn.manifold import TSNE\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.patches as mpatches\n",
        "import seaborn as sns\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7U4lApaW1y6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "ingredients = list()\n",
        "\n",
        "for recipe in data:\n",
        "    for ingredient in recipe[u'ingredients']:\n",
        "        ingredients.append(ingredient.strip())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LrkOZYkkXSGE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "eb7f82a6-2f2e-4e8a-c1e7-d4a3c90b0e66"
      },
      "source": [
        "print(\"Total Ingredients: \", len(ingredients))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total Ingredients:  9605394\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oLdYyR0gXUa9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bab83818-0457-41e7-c2e0-514284ee9f9c"
      },
      "source": [
        "\n",
        "print('unique ingredients in the dataset: ',len(set(ingredients)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "unique ingredients in the dataset:  2728692\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kuT7EBvIXYN7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ingredients_count = collections.Counter(ingredients)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DgI1YlZhXfAY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "38118924-d8b1-407f-8016-fa239f053599"
      },
      "source": [
        "ingredients_count.most_common(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('1 teaspoon salt', 50077),\n",
              " ('12 teaspoon salt', 46708),\n",
              " ('14 teaspoon salt', 28738),\n",
              " ('1 egg', 27971),\n",
              " ('2 eggs', 27494),\n",
              " ('salt and pepper', 25339),\n",
              " ('2 tablespoons olive oil', 24225),\n",
              " ('salt', 22948),\n",
              " ('1 teaspoon vanilla extract', 22012),\n",
              " ('1 tablespoon olive oil', 21628)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a81moiXDZg6Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "822e30b2-7d95-419a-a83b-905ccd7fb50a"
      },
      "source": [
        "\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "btj8XmemnsPQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del data[6784]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPgqaruEXmO_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "i = 0\n",
        "import inflect\n",
        "stemmer = inflect.engine()\n",
        "\n",
        "recipes_as_sentences_list = list()\n",
        "\n",
        "for recipe in data:\n",
        "    clean_recipe = list()\n",
        "    for ingredient in recipe['ingredients']:\n",
        "        ingredient = re.sub(r'[^a-zA-Z\\s]+', '', ingredient)\n",
        "        ingredient = re.sub(r'(?:^| )\\w(?:$| )', ' ', ingredient).strip()\n",
        "        #print(ingredient)\n",
        "        ingredient =  re.sub(r'\\(.*oz.\\)|sliced|fresh|large|medium|see|powder|packaged|package|size|layer|teaspoons|tsp|teaspoon|cut|diced|dice|note|tbsp|small|warm|plain|crushed|page|crumbles|cups|cup|ounces|ounce|ground|recipe|minced|chopped|sliced|tablespoons|tablespoon|chunks|chunk|liquid|bushels|bushel|dash|drops|drop|oz|gallons|gallon|glasses|glass|kgs|kg|liters|liter|ml|pinch|pints|pint|pounds|pound|quarts|quart|scoops|scoop|shots|shot','', ingredient)\n",
        "        #print(ingredient)\n",
        "        if len(ingredient)>=2:\n",
        "          if stemmer.singular_noun(ingredient) is not False:\n",
        "              ingredient = stemmer.singular_noun(ingredient)\n",
        "        words_in_ingredients = nltk.tag.pos_tag(ingredient.split())\n",
        "        #print(words_in_ingredients)\n",
        "        words_in_ingredients=[word[0] for word in words_in_ingredients if word[1] in ('NN','NNP')]\n",
        "        ingredient = \" \".join(words_in_ingredients)\n",
        "  \n",
        "        ingredient = ingredient.strip().lower()\n",
        "        clean_recipe.append(ingredient)\n",
        "    while(\"\" in clean_recipe) :\n",
        "        clean_recipe.remove(\"\")\n",
        "    recipes_as_sentences_list.append(clean_recipe)\n",
        "    print(i)\n",
        "    i+=1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "716WJywuYfhU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 595
        },
        "outputId": "5c592873-102e-4ad4-f9c3-10f11db4e709"
      },
      "source": [
        "data[7139]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '01c37cd29f',\n",
              " 'ingredients': ['1-1/2 tbsp. whole coriander seeds',\n",
              "  '1-1/2 tbsp. whole fennel seeds',\n",
              "  '1-1/2 tbsp. whole cumin',\n",
              "  '1-1/2 tbsp. whole black mustard seeds',\n",
              "  '1-1/2 tbsp. whole fenugreek',\n",
              "  '1-1/2 tbsp. whole black peppercorns',\n",
              "  '2-1/2 tbsp. ground ginger',\n",
              "  '2-1/2 tbsp. ground turmeric',\n",
              "  '1 cinnamon stick',\n",
              "  '5 cups callaloo, spinach',\n",
              "  'swiss chard, well-washed and chopped',\n",
              "  '1/4 cup of water',\n",
              "  '1 md. onion, diced',\n",
              "  '2 fresh tomatoes, diced',\n",
              "  '2 cloves of garlic, minced',\n",
              "  '1/4 Scotch bonnet, finely chopped',\n",
              "  '2 tbsp. of vegetable oil',\n",
              "  '1 tbsp. of butter',\n",
              "  'pinch of fresh',\n",
              "  'dried thyme',\n",
              "  'Salt to taste',\n",
              "  '1/4 cup vegetable oil',\n",
              "  '1-1/2 lb. braising',\n",
              "  'stewing beef, cut into 2-inch dices',\n",
              "  '1-1/2 cup onions, diced',\n",
              "  '1 cup West Indian white sweet potatoes, peeled and diced into 1-inch cubes',\n",
              "  '1 cup fresh',\n",
              "  'canned tomatoes',\n",
              "  '1-1/2 cup beef',\n",
              "  'chicken broth',\n",
              "  '1 cup of water',\n",
              "  '4 tbsp. of West Indian Curry',\n",
              "  'Salt']}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 182
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ibPZ6BINpcm8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "num_features = 30    # Word vector dimensionality                      \n",
        "min_word_count = 3    # 50% of the corpus                    \n",
        "num_workers = 4       # Number of CPUs\n",
        "context = 10          # Context window size;                                                                                \n",
        "downsampling = 1e-3   # threshold for configuring which \n",
        "                    # higher-frequency words are randomly downsampled\n",
        "\n",
        "# Initialize and train the model \n",
        "model = word2vec.Word2Vec(recipes_as_sentences_list, workers=num_workers, \\\n",
        "            size=num_features, min_count = min_word_count, \\\n",
        "            window = context, sample = downsampling)\n",
        "model.init_sims(replace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8SZQdMGpjvc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "486655a7-2e9d-4972-bb4c-db5de0f19a3f"
      },
      "source": [
        "model.most_similar(u'cheese')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('cheddar cheese', 0.810276985168457),\n",
              " ('lb elbow macaroni', 0.789874255657196),\n",
              " ('cup cheese', 0.7852275371551514),\n",
              " ('bread', 0.7805023193359375),\n",
              " ('cup ricotta cheese', 0.7796869874000549),\n",
              " ('ounce package spinach', 0.7770755887031555),\n",
              " ('ham', 0.767454206943512),\n",
              " ('ounce packet soup mix', 0.7605202794075012),\n",
              " ('spinach dry', 0.757777988910675),\n",
              " ('cup halfandhalf cream', 0.7573579549789429)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    }
  ]
}