Text Analytics - Tagging and Categorizing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "nbpresent": {
     "id": "91babcc8-bcce-45a9-ba25-138c632744be"
    }
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "text = 'This is just an example sentence for your reference'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "nbpresent": {
     "id": "078ff30c-78c9-47d7-8096-16a7f00253dd"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This', 'is', 'just', 'an', 'example', 'sentence', 'for', 'your', 'reference']"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk import word_tokenize\n",
    "words = word_tokenize(text)\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "nbpresent": {
     "id": "4bd5ba1e-eca1-41eb-8216-757e0a14402e"
    }
   },
   "outputs": [],
   "source": [
    "# Attaching a speech tag to each word\n",
    "tags = nltk.pos_tag(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "nbpresent": {
     "id": "c7aff6d7-c757-4fca-84d4-b3951a3cc2fd"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'a': 2, 'b': 1})"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To count words, count words with specific properties, count sequences and count tags, nltk defines the class:\n",
    "# nltk.FreqDist\n",
    "from nltk.probability import FreqDist\n",
    "\n",
    "# example\n",
    "list = ['a','b','a']\n",
    "fdist = FreqDist(list)\n",
    "fdist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "nbpresent": {
     "id": "2476bc99-4814-4597-aa95-ca5d60faa5ef"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist['a'] # 'a' is present twice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "nbpresent": {
     "id": "b3a34124-77de-4ec6-9f40-7f9b37d601e5"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist['c'] # we see 'c' 0 times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "nbpresent": {
     "id": "d68658cf-9342-4ae5-a340-3662a9d05649"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist.max() # the maximum seen letter is 'a'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "nbpresent": {
     "id": "4220b847-0b06-4b4e-9303-761cd7143c4e"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fdist) # we observed 2 types of objects in the stream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "nbpresent": {
     "id": "966b8e7a-f3f2-40a8-b8b4-093b62c9e2f7"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['a', 'b'])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist.keys() # Return the types of the objects that were observed in the stream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "nbpresent": {
     "id": "40ffc16c-6f71-4f01-978a-472a797ae42e"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666666"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist.freq('a') # 2/3 of the samples we saw were 'a'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "nbpresent": {
     "id": "77d9c55d-e428-4efc-834e-b656bce2d380"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdist.N() # How many samples did we count?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "nbpresent": {
     "id": "5278bb1b-bec6-4f2c-b770-8ea70f9e8e93"
    }
   },
   "outputs": [],
   "source": [
    "# NLTK contains a collection of tagged corpora. They are arranged as Python objects for the purpose of convenience.\n",
    "# I have used brown corpus in this example. The tagged_sents version is a list of sentences.\n",
    "# Each sentence is a list of pairs (tuples)(word,tag)\n",
    "# Corpus can also be accessed as a flat list of tagged words\n",
    "\n",
    "from nltk.corpus import brown\n",
    "brown_news_tagged = brown.tagged_sents(tagset='universal')\n",
    "brown_news_words = brown.tagged_words(tagset='universal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "nbpresent": {
     "id": "4363b49b-616b-47ae-9bd8-706e7f6718b7"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1161192"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdistw = FreqDist([w for (w, t) in brown_news_words])\n",
    "fdistw.N()    # We saw 1,161,192 words in this section of the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "nbpresent": {
     "id": "35454dd7-c44c-47ab-bbef-1105096d6e5c"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56057"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fdistw)  # How many distinct words are there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "nbpresent": {
     "id": "1200d59f-e482-4467-972b-c1b7b76848db"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdistw.max()  # What is the most frequent word?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "nbpresent": {
     "id": "a620749c-8239-460c-bcb7-7efea1c19dba"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "62713"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdistw['the']   # How often does 'the' occur in the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "nbpresent": {
     "id": "bb88c8e3-4275-4651-a4f4-a3862b8b59e0"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 5.40%\n"
     ]
    }
   ],
   "source": [
    "# frequency distribution of 'the'\n",
    "print('%5.2f%%' % (fdistw.freq('the') * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "nbpresent": {
     "id": "828b2500-ab6b-402f-8782-c342aa1c40b8"
    }
   },
   "outputs": [],
   "source": [
    "# Process textual data using TF-IDF\n",
    "s1 = 'The car is driven on the road'\n",
    "s2 = 'The truck is driven on the highway'\n",
    "s3 = 'The plane is flying in the air'\n",
    "s4 = 'People in Austin love keeping cats'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "nbpresent": {
     "id": "6381ed13-1007-4fd4-8b71-300f4dbcec3a"
    }
   },
   "outputs": [],
   "source": [
    "# Import Tfidf vectorizer from sklearn\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "nbpresent": {
     "id": "568c22a2-e5d7-44d8-ade1-8c982f0f44c9"
    }
   },
   "outputs": [],
   "source": [
    "# Initialize the vectorizer and then call fit and transform over it to calculate the TF-IDF score of the text\n",
    "vectorizer = TfidfVectorizer()\n",
    "response = vectorizer.fit_transform([s1,s2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "nbpresent": {
     "id": "348aac0b-710d-4eaf-9cab-41212676d1d7"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 6)\t0.604379551537\n",
      "  (0, 0)\t0.42471718587\n",
      "  (0, 3)\t0.302189775769\n",
      "  (0, 1)\t0.302189775769\n",
      "  (0, 4)\t0.302189775769\n",
      "  (0, 5)\t0.42471718587\n",
      "  (1, 6)\t0.604379551537\n",
      "  (1, 3)\t0.302189775769\n",
      "  (1, 1)\t0.302189775769\n",
      "  (1, 4)\t0.302189775769\n",
      "  (1, 7)\t0.42471718587\n",
      "  (1, 2)\t0.42471718587\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "nbpresent": {
     "id": "101b2862-8fa6-4d48-9493-71cb4adf2b77"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 1 0 0 0 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 1 0 1 0 0 1 0 1 1 0 1 0 0 0 1 0 1 0 0 0 0 0 0 1 1]\n",
      " [0 1 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 1 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 1 0 1 0 0 1 0 0 1 0 1 0 1 0 1 1 0 0]]\n",
      "{'all': 0, 'my': 11, 'cats': 2, 'in': 7, 'row': 14, 'when': 25, 'cat': 1, 'sits': 17, 'down': 3, 'she': 15, 'looks': 9, 'like': 8, 'furby': 6, 'toy': 24, 'the': 21, 'from': 5, 'outer': 12, 'space': 19, 'sunshine': 20, 'loves': 10, 'to': 23, 'sit': 16, 'this': 22, 'for': 4, 'some': 18, 'reason': 13}\n"
     ]
    }
   ],
   "source": [
    "# Bag of words \n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "corpus = ['All my cats in a row','When my cat sits down, she looks like a Furby toy!','The cat from outer space','Sunshine loves to sit like this for some reason.']\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "print( vectorizer.fit_transform(corpus).todense() )\n",
    "print( vectorizer.vocabulary_ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "nbpresent": {
     "id": "1fa48294-8842-4bfd-b2cb-d4963a8615eb"
    }
   },
   "outputs": [],
   "source": [
    "# Naive Bayes\n",
    "train = [('I love this sandwich.', 'pos'),\n",
    "('This is an amazing place!', 'pos'),\n",
    "('I feel very good about these beers.', 'pos'),\n",
    "('This is my best work.', 'pos'),\n",
    "(\"What an awesome view\", 'pos'),\n",
    "('I do not like this restaurant', 'neg'),\n",
    "('I am tired of this stuff.', 'neg'),\n",
    "(\"I can't deal with this\", 'neg'),\n",
    "('He is my sworn enemy!', 'neg'),\n",
    "('My boss is horrible.', 'neg')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "nbpresent": {
     "id": "00ed9958-bd79-461a-a87e-8a6abd09bfbd"
    }
   },
   "outputs": [],
   "source": [
    "# The first element of each tuple should be a dictionary of features. \n",
    "# So change the list into a data structure that the classifier can work with\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "nbpresent": {
     "id": "4a62177f-fcc5-4b52-96dd-39ec0790f158"
    }
   },
   "outputs": [],
   "source": [
    "all_words = set(word.lower() for passage in train for word in word_tokenize(passage[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "nbpresent": {
     "id": "6d164c49-0c5a-4641-bb36-0ccb7cc2efa9"
    }
   },
   "outputs": [],
   "source": [
    "t = [({word: (word in word_tokenize(x[0])) for word in all_words}, x[1]) for x in train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "nbpresent": {
     "id": "9fcd94bb-bb55-4d33-b71e-39e01df93bc2"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[({'!': False,\n",
       "   '.': True,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': False,\n",
       "   'love': True,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': True,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': True,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'pos'),\n",
       " ({'!': True,\n",
       "   '.': False,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': True,\n",
       "   'an': True,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': True,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': True,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'pos'),\n",
       " ({'!': False,\n",
       "   '.': True,\n",
       "   'about': True,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': True,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': True,\n",
       "   'good': True,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': True,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': True,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'pos'),\n",
       " ({'!': False,\n",
       "   '.': True,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': True,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': True,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': True,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': True},\n",
       "  'pos'),\n",
       " ({'!': False,\n",
       "   '.': False,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': True,\n",
       "   'awesome': True,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': True,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'pos'),\n",
       " ({'!': False,\n",
       "   '.': False,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': True,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': True,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': True,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': True,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': True,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'neg'),\n",
       " ({'!': False,\n",
       "   '.': True,\n",
       "   'about': False,\n",
       "   'am': True,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': True,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': True,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': True,\n",
       "   'tired': True,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'neg'),\n",
       " ({'!': False,\n",
       "   '.': False,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': True,\n",
       "   'deal': True,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': False,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": True,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': True,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': True,\n",
       "   'work': False},\n",
       "  'neg'),\n",
       " ({'!': True,\n",
       "   '.': False,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': False,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': True,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': False,\n",
       "   'i': False,\n",
       "   'is': True,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': True,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': True,\n",
       "   'these': False,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'neg'),\n",
       " ({'!': False,\n",
       "   '.': True,\n",
       "   'about': False,\n",
       "   'am': False,\n",
       "   'amazing': False,\n",
       "   'an': False,\n",
       "   'awesome': False,\n",
       "   'beers': False,\n",
       "   'best': False,\n",
       "   'boss': True,\n",
       "   'ca': False,\n",
       "   'deal': False,\n",
       "   'do': False,\n",
       "   'enemy': False,\n",
       "   'feel': False,\n",
       "   'good': False,\n",
       "   'he': False,\n",
       "   'horrible': True,\n",
       "   'i': False,\n",
       "   'is': True,\n",
       "   'like': False,\n",
       "   'love': False,\n",
       "   'my': False,\n",
       "   \"n't\": False,\n",
       "   'not': False,\n",
       "   'of': False,\n",
       "   'place': False,\n",
       "   'restaurant': False,\n",
       "   'sandwich': False,\n",
       "   'stuff': False,\n",
       "   'sworn': False,\n",
       "   'these': False,\n",
       "   'this': False,\n",
       "   'tired': False,\n",
       "   'very': False,\n",
       "   'view': False,\n",
       "   'what': False,\n",
       "   'with': False,\n",
       "   'work': False},\n",
       "  'neg')]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data is now structures\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "nbpresent": {
     "id": "09ba8e26-aaf6-4887-a944-772f62bee0e3"
    }
   },
   "outputs": [],
   "source": [
    "# The first element of each tuple is now a dictionary. The classifier can be trained now.\n",
    "classifier = nltk.NaiveBayesClassifier.train(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "nbpresent": {
     "id": "446cc40c-14ce-40c6-86a1-d96ca481f603"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                    this = True              neg : pos    =      2.3 : 1.0\n",
      "                    this = False             pos : neg    =      1.8 : 1.0\n",
      "                      an = False             neg : pos    =      1.6 : 1.0\n",
      "                       . = True              pos : neg    =      1.4 : 1.0\n",
      "                       . = False             neg : pos    =      1.4 : 1.0\n",
      "                      am = False             pos : neg    =      1.2 : 1.0\n",
      "                sandwich = False             neg : pos    =      1.2 : 1.0\n",
      "                    very = False             neg : pos    =      1.2 : 1.0\n",
      "                horrible = False             pos : neg    =      1.2 : 1.0\n",
      "                    love = False             neg : pos    =      1.2 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier.show_most_informative_features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "nbpresent": {
     "id": "7cf029ba-f8c8-4a32-9889-608571bf634f"
    }
   },
   "outputs": [],
   "source": [
    "# If you want to use the classifier, you can do it like this. First, begin with a test sentence:\n",
    "\n",
    "test_sentence = \"This is the best band I've ever heard!\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "nbpresent": {
     "id": "96bc024c-baf8-44ca-8777-5b5256f526f6"
    }
   },
   "outputs": [],
   "source": [
    "# Tokenize the sentence and figure out which words the sentence shares with all_words. These constitute the sentence's features.\n",
    "\n",
    "test_sent_features = {word.lower(): (word in word_tokenize(test_sentence.lower())) for word in all_words}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "nbpresent": {
     "id": "4a82ee71-e7ff-4cf9-bc89-27a19c7f4946"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'!': True,\n",
       " '.': False,\n",
       " 'about': False,\n",
       " 'am': False,\n",
       " 'amazing': False,\n",
       " 'an': False,\n",
       " 'awesome': False,\n",
       " 'beers': False,\n",
       " 'best': True,\n",
       " 'boss': False,\n",
       " 'ca': False,\n",
       " 'deal': False,\n",
       " 'do': False,\n",
       " 'enemy': False,\n",
       " 'feel': False,\n",
       " 'good': False,\n",
       " 'he': False,\n",
       " 'horrible': False,\n",
       " 'i': True,\n",
       " 'is': True,\n",
       " 'like': False,\n",
       " 'love': False,\n",
       " 'my': False,\n",
       " \"n't\": False,\n",
       " 'not': False,\n",
       " 'of': False,\n",
       " 'place': False,\n",
       " 'restaurant': False,\n",
       " 'sandwich': False,\n",
       " 'stuff': False,\n",
       " 'sworn': False,\n",
       " 'these': False,\n",
       " 'this': True,\n",
       " 'tired': False,\n",
       " 'very': False,\n",
       " 'view': False,\n",
       " 'what': False,\n",
       " 'with': False,\n",
       " 'work': False}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is how the features look like\n",
    "test_sent_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "nbpresent": {
     "id": "4121f8cb-b245-49b8-a428-19aab2e353fd"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pos'"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now classify the features\n",
    "classifier.classify(test_sent_features)   # This sentence appears to be positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "nbpresent": {
     "id": "664df527-313a-4b89-8869-5c11ddb825fb"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}