Spark Introduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MapReduce as a programming model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A Mapper function: \n",
    "Applies to every element in the collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fun(x):\n",
    "    return x**2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A Reducer function: An arbitrary associative binary operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add(x,y):\n",
    "    return x+y\n",
    "\n",
    "def maximum(x,y):\n",
    "    return max(x,y)\n",
    "\n",
    "def minimum(x,y):\n",
    "    return min(x,y)\n",
    "\n",
    "def prod(x,y):\n",
    "    return x*y\n",
    "\n",
    "def selective_division(x,y):\n",
    "    return x/y if x>y else y/x "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Conventional sequential program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79\n"
     ]
    }
   ],
   "source": [
    "L = [1,3,4,2,7]\n",
    "\n",
    "s = 0\n",
    "for a in L:\n",
    "    s = s+a**2\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The same calculation using the __map__ primitive. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 9, 16, 4, 49]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "79"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "    \n",
    "S = map(fun, L)\n",
    "print(S)\n",
    "reduce(add, S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No need to store the intermediate result: opens up the possibility for lazy evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce(add, map(fun, L))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selective_division(89, 65)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 9, 16, 4, 49]\n",
      "49\n"
     ]
    }
   ],
   "source": [
    "#print(L)\n",
    "print(S)\n",
    "\n",
    "#print reduce(selective_division, S)\n",
    "\n",
    "#print reduce(add, map(fun, L))\n",
    "\n",
    "#print reduce(minimum, S)\n",
    "print reduce(maximum, S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of even entries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_even(x):\n",
    "    return 1 if x%2==0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce(add, map(is_even, L))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating the average:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one(x):\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.4"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce(add, L)/float(reduce(add, map(one, L)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = ['Ahmet', 'Mehmet', 'Veli', 'Aras'] \n",
    "\n",
    "def is_A(x):\n",
    "    return 1 if x[0]=='A' else 0\n",
    "\n",
    "reduce(lambda x,y: x+y, map(is_A, A))\n",
    "\n",
    "#reduce(add, A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_and_add(x,y):\n",
    "    print(x,y)\n",
    "    return x+y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 1)\n",
      "(1, 2)\n",
      "(3, 3)\n",
      "(6, 4)\n",
      "(10, 5)\n",
      "(15, 6)\n",
      "(21, 7)\n",
      "(28, 8)\n",
      "(36, 9)\n",
      "(45, 10)\n",
      "(55, 11)\n",
      "(66, 12)\n",
      "(78, 13)\n",
      "(91, 14)\n",
      "(105, 15)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = range(16)\n",
    "\n",
    "reduce(print_and_add, L)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spark\n",
    "\n",
    "RDD : Resilient Distributed Dataset\n",
    "\n",
    "Note: This notebook version needs the Python 2.7. When the notebook server is launched from\n",
    "a 3.x environment, the environment variables are not set correctly."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a text file on the HDFS as a RDD\n",
    "* Display number of lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133228"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import findspark\n",
    "#findspark.init()\n",
    "#import pyspark\n",
    "#sc = pyspark.SparkContext(appName=\"Deneme\")\n",
    "\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "#textFile = sc.textFile(\"data/books-eng/hamlet.txt\")\n",
    "textFile = sc.textFile(\"data/books-eng\")\n",
    "textFile.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the first line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'ACT I'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find number of lines where a word appears"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = \"CLAUDIUS\"\n",
    "textFile.filter(lambda lin: word in lin).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get a random sample from the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'The office and devotion of their view'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.sample(withReplacement=False, fraction=0.05).first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Flourish. Enter ANTONY, CLEOPATRA, her Ladies, the Train, with Eunuchs fanning her',\n",
       " u'Look, where they come:',\n",
       " u'CLEOPATRA ',\n",
       " u'Then must thou needs find out new heaven, new earth.',\n",
       " u'Grates me: the sum.',\n",
       " u'MARK ANTONY ',\n",
       " u\"There's not a minute of our lives should stretch\",\n",
       " u'CLEOPATRA ',\n",
       " u'The qualities of people. Come, my queen;',\n",
       " u'Exeunt MARK ANTONY and CLEOPATRA with their train']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def prnt(x):\n",
    "    print x\n",
    "\n",
    "textFile.sample(withReplacement=False, fraction=0.05).take(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'ACT I',\n",
       " u\"SCENE I. Alexandria. A room in CLEOPATRA's palace.\",\n",
       " u'',\n",
       " u'Enter DEMETRIUS and PHILO ',\n",
       " u'PHILO ',\n",
       " u\"Nay, but this dotage of our general's\",\n",
       " u\"O'erflows the measure: those his goodly eyes,\",\n",
       " u\"That o'er the files and musters of the war\",\n",
       " u\"Have glow'd like plated Mars, now bend, now turn,\",\n",
       " u'The office and devotion of their view']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.take(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the line with most words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.map(lambda line: len(line.split())).reduce(lambda a, b: a if (a > b) else b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(122,\n",
       " u\"    [Enter a King and a Queen very lovingly; the Queen embracing him, and he her. She kneels, and makes show of protestation unto him. He takes her up, and declines his head upon her neck: lays him down upon a bank of flowers: she, seeing him asleep, leaves him. Anon comes in a fellow, takes off his crown, kisses it, and pours poison in the King's ears, and exit. The Queen returns; finds the King dead, and makes passionate action. The Poisoner, with some two or three Mutes, comes in again, seeming to lament with her. The dead body is carried away. The Poisoner wooes the Queen with gifts: she seems loath and unwilling awhile, but in the end accepts his love]\")"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.map(lambda line: (len(line.split()),line) ).reduce(lambda a, b: a if (a[0] > b[0]) else b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abc', 'df']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'abc df'.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = 'xyz'\n",
    "pair = (len(s), s)\n",
    "\n",
    "pair[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Exasperated,', 2),\n",
       " (u'aided', 2),\n",
       " (u'drunkenness.', 2),\n",
       " (u'glad,\"', 5),\n",
       " (u'Chichagov,', 8),\n",
       " (u\"unvarnish'd\", 1),\n",
       " (u'Danois', 1),\n",
       " (u'admiration?', 1),\n",
       " (u'forges', 3),\n",
       " (u'blisters', 2),\n",
       " (u'habilet\\xe9;', 1),\n",
       " (u'brightening', 4),\n",
       " (u'ever?', 1),\n",
       " (u'travaux', 2),\n",
       " (u'Romeo,--', 1),\n",
       " (u'Kirilovich', 1),\n",
       " (u'pedestrian,', 1),\n",
       " (u'bourreau.', 1),\n",
       " (u'party.', 10),\n",
       " (u'rocking', 6),\n",
       " (u'piti\\xe9!--Elle', 1),\n",
       " (u'booths.', 1),\n",
       " (u'looking', 415),\n",
       " (u'flash.', 3),\n",
       " (u'Senior', 1),\n",
       " (u'pointing', 78),\n",
       " (u'inheritance.', 2),\n",
       " (u'Sewer,', 1),\n",
       " (u'(His', 1),\n",
       " (u'larger,', 1),\n",
       " (u'contained', 7),\n",
       " (u\"reckon'd.\", 1),\n",
       " (u'Oka', 1),\n",
       " (u'mark-man!', 1),\n",
       " (u'donner?', 1),\n",
       " (u'Eh,', 7),\n",
       " (u'war!\"', 1),\n",
       " (u'Craftsmen', 1),\n",
       " (u'\"Encouraging', 1),\n",
       " (u'attack,\"', 2),\n",
       " (u'enclosures,', 1),\n",
       " (u'1813.', 2),\n",
       " (u'nourisher', 1),\n",
       " (u'artilleryman', 4),\n",
       " (u'motifs', 1),\n",
       " (u'cozenage', 1),\n",
       " (u'be!\"', 12),\n",
       " (u'today?', 3),\n",
       " (u'trumpeters,', 1),\n",
       " (u'blanket', 3),\n",
       " (u'breadth.', 2),\n",
       " (u'goats,', 2),\n",
       " (u'particulars.', 1),\n",
       " (u'powder.', 2),\n",
       " (u'CHRIS', 10),\n",
       " (u'smack', 2),\n",
       " (u'pacify', 2),\n",
       " (u'yellow', 25),\n",
       " (u'voix', 8),\n",
       " (u'patrie,', 1),\n",
       " (u'prodigies,', 1),\n",
       " (u'Ottoman.', 1),\n",
       " (u'postpone', 3),\n",
       " (u'vast', 36),\n",
       " (u'Lest', 15),\n",
       " (u'revelers', 1),\n",
       " (u'present!', 1),\n",
       " (u'baking', 1),\n",
       " (u'destitution,', 1),\n",
       " (u'skills', 2),\n",
       " (u'Pierre:', 5),\n",
       " (u'(R2-D2).........................................KENNY', 2),\n",
       " (u'spacecraft', 2),\n",
       " (u'relentlessly', 1),\n",
       " (u'ending,', 2),\n",
       " (u'blowing,', 1),\n",
       " (u'ivrogne.', 1),\n",
       " (u\"hack'd\", 1),\n",
       " (u'gracious:', 1),\n",
       " (u'RODERIGO.--Quoi!', 1),\n",
       " (u'stretchers.', 1),\n",
       " (u'glued', 2),\n",
       " (u'satisfaire,', 1),\n",
       " (u'stars!', 2),\n",
       " (u'anywhere!\"', 1),\n",
       " (u'\"mon', 2),\n",
       " (u'parted', 16),\n",
       " (u'Mamma.\"', 3),\n",
       " (u'Draws', 2),\n",
       " (u'scouted', 1),\n",
       " (u'BREMNER', 1),\n",
       " (u'first...', 1),\n",
       " (u'warns', 2),\n",
       " (u'smeared', 8),\n",
       " (u'parlons', 2),\n",
       " (u\"Bitski's\", 1),\n",
       " (u'Annette', 2),\n",
       " (u'equal,', 1),\n",
       " (u'blaspheme', 1),\n",
       " (u'rescript', 5),\n",
       " (u'deprived.', 1),\n",
       " (u'purged,', 2),\n",
       " (u'anybody,', 1),\n",
       " (u'cantina,', 2),\n",
       " (u'rope', 6),\n",
       " (u'gold!', 1),\n",
       " (u'him--felt', 1),\n",
       " (u'hide', 60),\n",
       " (u'children?\"', 1),\n",
       " (u'records,', 1),\n",
       " (u'CATO,', 2),\n",
       " (u'beeping.', 1),\n",
       " (u'Spacecraft', 1),\n",
       " (u'finesse;', 1),\n",
       " (u'victims', 3),\n",
       " (u'astonishment.', 4),\n",
       " (u'unpleasant,', 1),\n",
       " (u'better;', 1),\n",
       " (u'See-Threepio', 4),\n",
       " (u'subordinate,', 1),\n",
       " (u'majesties', 2),\n",
       " (u'difficulty.', 8),\n",
       " (u'tips', 6),\n",
       " (u'supple,', 1),\n",
       " (u'grunting', 2),\n",
       " (u'kid.', 11),\n",
       " (u'suave,', 1),\n",
       " (u'insinuation', 1),\n",
       " (u'Stolypin', 2),\n",
       " (u'hero', 17),\n",
       " (u'10,', 1),\n",
       " (u'Conceit', 2),\n",
       " (u'Oh...', 3),\n",
       " (u'Amuse', 1),\n",
       " (u'feisty', 1),\n",
       " (u'Fortuna..............................................MICHAEL', 1),\n",
       " (u'(raw', 1),\n",
       " (u'wife--are', 1),\n",
       " (u'Brother.', 1),\n",
       " (u'backs,', 3),\n",
       " (u'darling....', 1),\n",
       " (u'glass.', 15),\n",
       " (u'clenching', 3),\n",
       " (u'battery?', 1),\n",
       " (u'shriek', 4),\n",
       " (u'beloved;', 1),\n",
       " (u'Fili', 4),\n",
       " (u'quieter,\"', 1),\n",
       " (u'gests.', 1),\n",
       " (u'bellow.', 1),\n",
       " (u'glories;', 1),\n",
       " (u'bedding', 1),\n",
       " (u'worse.', 13),\n",
       " (u'Antonius,', 1),\n",
       " (u'Mary--\"Haven\\'t', 1),\n",
       " (u'fr\\xe9n\\xe9sie:--Tout', 1),\n",
       " (u'MAGIC', 2),\n",
       " (u'DESD\\xc9MONA.--Ma', 2),\n",
       " (u'selvedges.', 1),\n",
       " (u'sombre', 3),\n",
       " (u'smoke', 90),\n",
       " (u'NOT', 4),\n",
       " (u'PORTMAN', 1),\n",
       " (u'settled', 54),\n",
       " (u'says,\"', 2),\n",
       " (u'arbiters,\"', 1),\n",
       " (u'punctilious', 1),\n",
       " (u'circumstance.', 1),\n",
       " (u'ridiculed', 2),\n",
       " (u'sugar.', 1),\n",
       " (u'Mantua;', 2),\n",
       " (u'straw', 12),\n",
       " (u'd\\xe9sire', 1),\n",
       " (u'carack:', 1),\n",
       " (u'Were,', 1),\n",
       " (u\"KENOBI'S\", 1),\n",
       " (u'Pokrovsk,', 2),\n",
       " (u'us?!', 1),\n",
       " (u'Master!...\"', 1),\n",
       " (u'for--I', 1),\n",
       " (u'resonant', 1),\n",
       " (u'v\\xe9rit\\xe9,', 16),\n",
       " (u'breath?', 4),\n",
       " (u'reassured.', 1),\n",
       " (u'anyhow', 3),\n",
       " (u'rareness,', 1),\n",
       " (u'Wookiee', 35),\n",
       " (u'Star', 139),\n",
       " (u'patchwork', 1),\n",
       " (u'leurs', 29),\n",
       " (u'XXXVIII', 1),\n",
       " (u'exacte', 1),\n",
       " (u'dispatches', 2),\n",
       " (u'hospital', 18),\n",
       " (u'exhort', 2),\n",
       " (u\"diplomat's\", 1),\n",
       " (u'blacksmiths', 1),\n",
       " (u'exception;', 2),\n",
       " (u'backing', 1),\n",
       " (u\"saw't:\", 1),\n",
       " (u'crafty,', 1),\n",
       " (u'strike', 35),\n",
       " (u'chewing', 4),\n",
       " (u'trot.', 4),\n",
       " (u'(softly)', 2),\n",
       " (u'used,', 3),\n",
       " (u'circonstance', 1),\n",
       " (u'presided', 1),\n",
       " (u'higher,', 5),\n",
       " (u'instructions...', 1),\n",
       " (u'property--that', 1),\n",
       " (u'barbarous', 1),\n",
       " (u'\"Pretty,', 1),\n",
       " (u'unconsciousness.', 1),\n",
       " (u'ronflent;', 1),\n",
       " (u'jiggles', 1),\n",
       " (u'meat,', 5),\n",
       " (u\"JAGO.--N'en\", 1),\n",
       " (u'mettant', 1),\n",
       " (u'off?', 1),\n",
       " (u'Never!\"', 1),\n",
       " (u'IAGO', 288),\n",
       " (u'GARTIN,', 1),\n",
       " (u'troops.', 37),\n",
       " (u'Likhachev;', 1),\n",
       " (u'ouvrent', 1),\n",
       " (u'bow.', 3),\n",
       " (u'FIGHTER', 38),\n",
       " (u'guest:', 1),\n",
       " (u'sped', 1),\n",
       " (u'DEBORAH', 1),\n",
       " (u'Settlement.', 1),\n",
       " (u'EVEN', 2),\n",
       " (u'Jena', 4),\n",
       " (u'word', 220),\n",
       " (u'uncovered', 4),\n",
       " (u'yourself....\"', 3),\n",
       " (u'40', 1),\n",
       " (u'T-forty-', 1),\n",
       " (u'famed', 2),\n",
       " (u'Crown', 1),\n",
       " (u'laughter;', 1),\n",
       " (u'Copenhagen--a', 1),\n",
       " (u'majesty;', 2),\n",
       " (u'respect:', 2),\n",
       " (u'choose,', 2),\n",
       " (u'\"Impudence!', 1),\n",
       " (u'dispirited.', 1),\n",
       " (u'underlings.', 1),\n",
       " (u'criticizing', 3),\n",
       " (u'ruddy,', 1),\n",
       " (u'Guildenstern;', 1),\n",
       " (u'haters.', 1),\n",
       " (u'vicomte,\"', 1),\n",
       " (u'shore!', 1),\n",
       " (u'\\xe9tiez', 1),\n",
       " (u'\"Voyons,', 1),\n",
       " (u'\"Wherefore?\"', 1),\n",
       " (u'contented.', 2),\n",
       " (u'collision.', 1),\n",
       " (u'fichu', 2),\n",
       " (u'Charity,', 1),\n",
       " (u'\"H-o-o-w', 1),\n",
       " (u\"l'oreille\", 4),\n",
       " (u'Napoleon;', 2),\n",
       " (u'someone?--he', 1),\n",
       " (u'underclothing,', 1),\n",
       " (u'fair!', 1),\n",
       " (u'Quant', 3),\n",
       " (u'Holding', 7),\n",
       " (u'DAVIS', 2),\n",
       " (u'organizer', 1),\n",
       " (u'workbag.\"', 1),\n",
       " (u'blank,', 1),\n",
       " (u'Salomoni!', 1),\n",
       " (u'landlady,', 1),\n",
       " (u'ceremony', 12),\n",
       " (u'rites?', 1),\n",
       " (u'Darklighter', 1),\n",
       " (u'dismantled,', 1),\n",
       " (u'Feoktist?\"', 1),\n",
       " (u'LODOVICO.--Sa', 1),\n",
       " (u'night,--', 1),\n",
       " (u'machine', 20),\n",
       " (u'tuer.--En', 1),\n",
       " (u'ablaze.', 1),\n",
       " (u'desperately,', 6),\n",
       " (u'officier,', 1),\n",
       " (u'Recalling', 2),\n",
       " (u'd\\xe9mon!', 2),\n",
       " (u\"Tsarevich's\", 1),\n",
       " (u'ordinary', 36),\n",
       " (u'III', 26),\n",
       " (u'gentleman\"),', 1),\n",
       " (u\"letter's\", 1),\n",
       " (u'HORATIO,', 7),\n",
       " (u'reasonable.\"', 1),\n",
       " (u'laide', 2),\n",
       " (u\"infant's\", 2),\n",
       " (u'pure-white', 1),\n",
       " (u'Urusov', 1),\n",
       " (u'flair.', 1),\n",
       " (u'sullenly', 2),\n",
       " (u'CASSIO.--Tu', 1),\n",
       " (u'fellows:', 1),\n",
       " (u'Rostopchin,', 20),\n",
       " (u'Dialogue', 2),\n",
       " (u'Russen!', 1),\n",
       " (u'Apocalypse.', 1),\n",
       " (u'Will', 109),\n",
       " (u'hereupon', 1),\n",
       " (u'walkway,', 2),\n",
       " (u\"d'amiti\\xe9\", 1),\n",
       " (u'food;', 3),\n",
       " (u'misunderstandings,', 1),\n",
       " (u'domed', 4),\n",
       " (u'cuisse', 2),\n",
       " (u'chilled', 2),\n",
       " (u'cold;', 3),\n",
       " (u'ch\\xe8vre', 1),\n",
       " (u\"confess'd.\", 1),\n",
       " (u'Antony,', 50),\n",
       " (u'\"Despite', 1),\n",
       " (u'revolt', 4),\n",
       " (u'neck...', 1),\n",
       " (u'beasts,\"', 1),\n",
       " (u'unsmirched', 1),\n",
       " (u'chers', 2),\n",
       " (u'zoology', 1),\n",
       " (u'power...\"', 1),\n",
       " (u'burgher', 1),\n",
       " (u'Frenchie!', 1),\n",
       " (u'KNOWLES', 1),\n",
       " (u'amis', 3),\n",
       " (u'vo\\xfbte', 1),\n",
       " (u'opposition,\"', 1),\n",
       " (u'Eye', 2),\n",
       " (u'sun,', 22),\n",
       " (u'peur', 2),\n",
       " (u'endormie,', 1),\n",
       " (u'goggles.', 2),\n",
       " (u'unwrung.', 1),\n",
       " (u'\"Karay?', 1),\n",
       " (u'beau,', 1),\n",
       " (u'Moff', 8),\n",
       " (u'hibernation', 1),\n",
       " (u'surrendered', 9),\n",
       " (u'judge,\"', 1),\n",
       " (u'plated', 1),\n",
       " (u'quelquefois', 3),\n",
       " (u'Rapidly', 2),\n",
       " (u'miscarry', 1),\n",
       " (u'General,', 9),\n",
       " (u'exhibition,', 1),\n",
       " (u'luck!\"', 3),\n",
       " (u'glances', 33),\n",
       " (u'one-ruble', 1),\n",
       " (u'hobbledehoy', 1),\n",
       " (u'MARTY', 1),\n",
       " (u'merrier,', 1),\n",
       " (u'mettent', 1),\n",
       " (u'Attendants]', 11),\n",
       " (u'tears?', 3),\n",
       " (u'pains!', 1),\n",
       " (u'good-by', 6),\n",
       " (u'died;', 3),\n",
       " (u'bouquet', 1),\n",
       " (u'haquen\\xe9es', 1),\n",
       " (u'routine,', 1),\n",
       " (u'spy.', 2),\n",
       " (u'savory', 4),\n",
       " (u'pressing,', 1),\n",
       " (u'bon!', 1),\n",
       " (u'crunch.', 1),\n",
       " (u'anxiety.', 5),\n",
       " (u'peu:', 1),\n",
       " (u'starships', 2),\n",
       " (u'fortunate:', 1),\n",
       " (u'bog.', 2),\n",
       " (u'affected', 16),\n",
       " (u'manifesto.', 1),\n",
       " (u'fragment', 6),\n",
       " (u'vainglorious', 1),\n",
       " (u'malice,', 2),\n",
       " (u'Dunyasha....\"', 1),\n",
       " (u'MV-7.', 1),\n",
       " (u'meadow,', 6),\n",
       " (u'history.', 15),\n",
       " (u'HANSON,', 1),\n",
       " (u'JAN', 1),\n",
       " (u'Anon', 3),\n",
       " (u'Steps', 2),\n",
       " (u'sack', 3),\n",
       " (u'suitor', 7),\n",
       " (u'depression', 9),\n",
       " (u'confiant?', 1),\n",
       " (u'17th', 1),\n",
       " (u\"Othello_.--Qu'avez-vous\", 1),\n",
       " (u'senator.', 1),\n",
       " (u'manners,', 4),\n",
       " (u'exhaled', 1),\n",
       " (u'spotlighted', 1),\n",
       " (u'Fates', 1),\n",
       " (u'weather.', 3),\n",
       " (u'orderlies', 10),\n",
       " (u'mammas', 1),\n",
       " (u'Balashev', 71),\n",
       " (u'sold.\"', 1),\n",
       " (u'rendered,', 1),\n",
       " (u'WANNBERG', 2),\n",
       " (u'Potsdam,', 1),\n",
       " (u'Captains,', 1),\n",
       " (u\"donne-m'en\", 1),\n",
       " (u'OTHELLO.--Avez-vous', 2),\n",
       " (u'Decius,', 2),\n",
       " (u'organized', 5),\n",
       " (u'hems.', 1),\n",
       " (u'Uh,', 10),\n",
       " (u'WILLIAMS', 4),\n",
       " (u'\"great\"', 2),\n",
       " (u'twirled', 4),\n",
       " (u'lifted', 65),\n",
       " (u'tea?\"', 5),\n",
       " (u'Solicit', 1),\n",
       " (u'assistants.', 3),\n",
       " (u'Verona.', 1),\n",
       " (u'XC.', 1),\n",
       " (u'bowls', 3),\n",
       " (u'supposed.', 1),\n",
       " (u\"Walk'd\", 1),\n",
       " (u'Medicine', 2),\n",
       " (u'communications,', 1),\n",
       " (u'Fahrenheit', 1),\n",
       " (u'laugh?', 1),\n",
       " (u'seminude', 1),\n",
       " (u'machinant', 1),\n",
       " (u'deviennent', 1),\n",
       " (u'ghosts', 3),\n",
       " (u'appurtenance', 1),\n",
       " (u'distinguee,', 1),\n",
       " (u\"'Hurrah!'--a\", 1),\n",
       " (u'damne-toi,', 1),\n",
       " (u'confineless', 1),\n",
       " (u'\"Greatness,\"', 1),\n",
       " (u'langue', 2),\n",
       " (u'smoked,', 3),\n",
       " (u'at,', 7),\n",
       " (u'headdress', 2),\n",
       " (u'glaive!--Encore', 1),\n",
       " (u'refuge!', 1),\n",
       " (u'Institute', 1),\n",
       " (u'speed?', 1),\n",
       " (u'Animatronics', 1),\n",
       " (u'start,', 5),\n",
       " (u'lain', 8),\n",
       " (u\"Nothing's\", 1),\n",
       " (u'wooingly', 1),\n",
       " (u'bed.\"', 2),\n",
       " (u'question;', 1),\n",
       " (u'nitrogen', 1),\n",
       " (u'clink;', 1),\n",
       " (u'dummy!\"', 1),\n",
       " (u'jour!', 3),\n",
       " (u'supplies,', 2),\n",
       " (u'assured', 21),\n",
       " (u'derision.', 1),\n",
       " (u'bottommost', 1),\n",
       " (u'eternal', 23),\n",
       " (u'tourment\\xe9', 2),\n",
       " (u'atmosphere.\"', 1),\n",
       " (u'Discussions', 1),\n",
       " (u'indices', 1),\n",
       " (u'XXX.', 1),\n",
       " (u'steepy', 1),\n",
       " (u'springtime', 1),\n",
       " (u'cloth,', 3),\n",
       " (u'Guardsman,', 1),\n",
       " (u'contingencies', 4),\n",
       " (u\"l'amour\", 14),\n",
       " (u'knit', 10),\n",
       " (u'souveraine,', 1),\n",
       " (u'Retire-toi;', 1),\n",
       " (u\"if't\", 5),\n",
       " (u'Emperor--without', 1),\n",
       " (u'waddled', 2),\n",
       " (u\"know't;\", 1),\n",
       " (u\"roof'd,\", 1),\n",
       " (u'LOFTHOUSE', 1),\n",
       " (u'\"They\\'ve', 6),\n",
       " (u'us--namely', 1),\n",
       " (u'lour', 1),\n",
       " (u'raise,', 1),\n",
       " (u'brooding', 3),\n",
       " (u\"'Las,\", 1),\n",
       " (u'local', 5),\n",
       " (u'filter', 2),\n",
       " (u'renew', 8),\n",
       " (u'old--the', 1),\n",
       " (u'features,', 7),\n",
       " (u'cane.', 1),\n",
       " (u'Human', 2),\n",
       " (u'comptes-y', 1),\n",
       " (u'Truck', 1),\n",
       " (u'tricks?', 1),\n",
       " (u'tactics', 4),\n",
       " (u'\"Denisov!', 1),\n",
       " (u'gossips,', 2),\n",
       " (u'lathe.', 1),\n",
       " (u'Cooper', 3),\n",
       " (u'render', 12),\n",
       " (u'sea-bank', 1),\n",
       " (u'hillock.', 1),\n",
       " (u\"s'accommodera\", 1),\n",
       " (u'negative,', 1),\n",
       " (u'hour!', 2),\n",
       " (u'bombard', 2),\n",
       " (u'camps,', 4),\n",
       " (u'SPACECRAFT', 1),\n",
       " (u'\"Awfully', 1),\n",
       " (u'Field.', 2),\n",
       " (u'delicious', 4),\n",
       " (u'separation!', 1),\n",
       " (u'\"Marya', 4),\n",
       " (u'prolong\\xe9', 1),\n",
       " (u'justes', 1),\n",
       " (u'moment...\"', 1),\n",
       " (u'man--that', 1),\n",
       " (u'knight.', 1),\n",
       " (u'\"Wait--just', 1),\n",
       " (u'lisping:', 1),\n",
       " (u'Wanna', 2),\n",
       " (u'continuing', 15),\n",
       " (u'ha-ve', 1),\n",
       " (u'attack', 107),\n",
       " (u'MADINE', 1),\n",
       " (u'shipwrights,', 1),\n",
       " (u'impudente', 1),\n",
       " (u'NICHOLS,', 2),\n",
       " (u'Uncle', 25),\n",
       " (u'tune', 13),\n",
       " (u'corns;', 1),\n",
       " (u'd\\xe9p\\xeache-toi.', 1),\n",
       " (u'father...\"', 2),\n",
       " (u'purposefully', 2),\n",
       " (u'empty!', 1),\n",
       " (u'next--to', 1),\n",
       " (u\"headquarters.'\", 1),\n",
       " (u'unlearn', 1),\n",
       " (u'naught:', 2),\n",
       " (u'nightingale:', 1),\n",
       " (u'Morality.', 1),\n",
       " (u'moisture', 5),\n",
       " (u'Jobert', 2),\n",
       " (u'monotonous,', 1),\n",
       " (u'short-circuit.', 2),\n",
       " (u'travers', 2),\n",
       " (u'floor.', 27),\n",
       " (u'purses', 1),\n",
       " (u'prouder', 1),\n",
       " (u'Vaska,', 1),\n",
       " (u'Villanous', 2),\n",
       " (u'selected.', 2),\n",
       " (u'OTHELLO.--Allez', 2),\n",
       " (u'furlough', 3),\n",
       " (u'Wait.', 2),\n",
       " (u'battle--the', 1),\n",
       " (u'fiction,', 1),\n",
       " (u'positively', 7),\n",
       " (u'yours?\"', 3),\n",
       " (u'Him!...', 1),\n",
       " (u'targes', 1),\n",
       " (u'creatures,', 4),\n",
       " (u'plate,', 5),\n",
       " (u'twenty,', 2),\n",
       " (u'exist....', 1),\n",
       " (u'joke....\"', 1),\n",
       " (u'spurring', 3),\n",
       " (u\"d'hypocrisie\", 1),\n",
       " (u'hors', 5),\n",
       " (u'1977', 2),\n",
       " (u'body;', 1),\n",
       " (u'indecision,', 3),\n",
       " (u'guilt', 11),\n",
       " (u'boil', 3),\n",
       " (u'to,', 59),\n",
       " (u'Archduchy', 1),\n",
       " (u'audibly.', 1),\n",
       " (u'girlishly', 1),\n",
       " (u'talk--all', 1),\n",
       " (u'girl!', 5),\n",
       " (u'woman!\"', 5),\n",
       " (u'WIPPERT', 1),\n",
       " (u'we?', 6),\n",
       " (u'\"Heavens!', 2),\n",
       " (u\"Kutaysov's\", 1),\n",
       " (u'swallowed:', 1),\n",
       " (u'falcon\"', 1),\n",
       " (u'subsided.', 1),\n",
       " (u'CLEOPATRA', 215),\n",
       " (u'Pavlovna--who', 1),\n",
       " (u'bloodshot,', 1),\n",
       " (u'mari.', 7),\n",
       " (u'tasted.', 2),\n",
       " (u'Willarski,', 8),\n",
       " (u'shy,', 6),\n",
       " (u'nonpareil.', 1),\n",
       " (u'swindled', 1),\n",
       " (u'staggeringly', 1),\n",
       " (u'jam,', 1),\n",
       " (u'leafy', 2),\n",
       " (u'intellect,', 4),\n",
       " (u'wagered', 2),\n",
       " (u'moist.', 3),\n",
       " (u\"counterfeit'st\", 1),\n",
       " (u'else--in', 1),\n",
       " (u'Staying', 2),\n",
       " (u'wholesome', 10),\n",
       " (u'audacity!', 1),\n",
       " (u'POV', 1),\n",
       " (u'often,', 5),\n",
       " (u'railings,', 1),\n",
       " (u'Manager', 7),\n",
       " (u'uttermost?', 1),\n",
       " (u'ensemble.)', 1),\n",
       " (u'Emperor.', 54),\n",
       " (u\"pineapples.'\", 1),\n",
       " (u'doors', 34),\n",
       " (u'impalpable,', 1),\n",
       " (u'Foh!', 1),\n",
       " (u'r\\xe9p\\xe9tait', 1),\n",
       " (u'blessed,', 2),\n",
       " (u'completed', 8),\n",
       " (u'Directed', 2),\n",
       " (u'allow?', 1),\n",
       " (u'Convert', 2),\n",
       " (u'feverishly.', 1),\n",
       " (u'lui.--O', 1),\n",
       " (u'panicked.', 1),\n",
       " (u'Redoubt--and', 1),\n",
       " (u'Brutus:', 3),\n",
       " (u'Alarm', 1),\n",
       " (u'see;', 3),\n",
       " (u'report?', 1),\n",
       " (u'cordes', 1),\n",
       " (u'JEAN', 1),\n",
       " (u'hunts,', 3),\n",
       " (u\"l'emporte\", 1),\n",
       " (u'(Though', 1),\n",
       " (u'saule.', 3),\n",
       " (u'bees.', 2),\n",
       " (u'content;', 2),\n",
       " (u'freely.', 4),\n",
       " (u'Went', 2),\n",
       " (u'difficulties,', 1),\n",
       " (u'feeble,', 5),\n",
       " (u'grave', 32),\n",
       " (u'pillows.', 3),\n",
       " (u'hut', 35),\n",
       " (u'initiator', 1),\n",
       " (u'don', 3),\n",
       " (u'Immoderately', 1),\n",
       " (u'unripe,', 1),\n",
       " (u'Unthrifty', 1),\n",
       " (u'medical', 9),\n",
       " (u'm', 3),\n",
       " (u'retreated.', 2),\n",
       " (u\"d'accord,\", 1),\n",
       " (u'investing', 1),\n",
       " (u'courrouc\\xe9e', 1),\n",
       " (u'justifiability', 1),\n",
       " (u'Karl', 7),\n",
       " (u'another,\"', 5),\n",
       " (u\"me'?\", 1),\n",
       " (u'thresh', 1),\n",
       " (u'letters;', 1),\n",
       " (u'flog', 2),\n",
       " (u'incomplete', 1),\n",
       " (u'duets', 1),\n",
       " (u'ETERNITY', 1),\n",
       " (u'memory),', 1),\n",
       " (u'Napoleon,\"', 1),\n",
       " (u'bait', 4),\n",
       " (u'come.\"', 4),\n",
       " (u'wrathfully', 2),\n",
       " (u'insects', 2),\n",
       " (u'blazed', 1),\n",
       " (u'harness,', 5),\n",
       " (u'liest', 3),\n",
       " (u'him--evidently', 1),\n",
       " (u\"enjoin'd\", 1),\n",
       " (u'convulsively,', 1),\n",
       " (u'resuming', 3),\n",
       " (u'\"This,', 1),\n",
       " (u'ROSS,', 9),\n",
       " (u'appease', 2),\n",
       " (u'Anne', 1),\n",
       " (u'succeed.', 1),\n",
       " (u'Rostova?\"', 1),\n",
       " (u'lessons', 8),\n",
       " (u'ear.', 14),\n",
       " (u'submit', 14),\n",
       " (u'shot!', 2),\n",
       " (u'gantry.', 2),\n",
       " (u'vexations', 1),\n",
       " (u'Parle,', 1),\n",
       " (u'vilest,', 1),\n",
       " (u'Duchenois,', 1),\n",
       " (u'Bib', 19),\n",
       " (u'haste;', 4),\n",
       " (u'\"Going', 2),\n",
       " (u'mistake,', 6),\n",
       " (u'hurrah!', 4),\n",
       " (u'bush', 2),\n",
       " (u'\"Sideways!', 1),\n",
       " (u'Mary:', 1),\n",
       " (u'specify', 1),\n",
       " (u'everything...', 2),\n",
       " (u'dullness', 2),\n",
       " (u'stores', 8),\n",
       " (u'radically', 1),\n",
       " (u\"dispatch'd:\", 1),\n",
       " (u'reverently', 1),\n",
       " (u'precedence;', 1),\n",
       " (u\"Kutuzov's.\", 1),\n",
       " (u'\"Enchantress,\"', 1),\n",
       " (u'came;--with', 1),\n",
       " (u'ENOBARBUS]', 4),\n",
       " (u'predestined', 8),\n",
       " (u'\"Weren\\'t', 1),\n",
       " (u'fruits,', 1),\n",
       " (u'lies.', 8),\n",
       " (u'view:', 1),\n",
       " (u\"Capel's\", 2),\n",
       " (u'(Yoda', 1),\n",
       " (u'comply', 17),\n",
       " (u'malgr\\xe9', 1),\n",
       " (u'maintenant?--Malheureuse', 1),\n",
       " (u'debt:', 2),\n",
       " (u'Fife', 1),\n",
       " (u'elope', 5),\n",
       " (u'men:', 5),\n",
       " (u'noon', 4),\n",
       " (u'post-horses;', 1),\n",
       " (u'\"Love...', 1),\n",
       " (u'Grandeur', 1),\n",
       " (u'dipping', 2),\n",
       " (u'flight?', 1),\n",
       " (u'thrusts', 3),\n",
       " (u'remembers...', 1),\n",
       " (u'Norway;', 1),\n",
       " (u'stead', 1),\n",
       " (u'reviving', 1),\n",
       " (u'flanked', 4),\n",
       " (u\"Vasili's\", 25),\n",
       " (u'advance.', 14),\n",
       " (u'\"man', 4),\n",
       " (u\"'overresist'\", 1),\n",
       " (u'Montague!', 2),\n",
       " (u\"Wha-wha-what's\", 1),\n",
       " (u'fascinating', 7),\n",
       " (u'impotent', 2),\n",
       " (u'yawned', 1),\n",
       " (u'admiral,', 1),\n",
       " (u'Seslavin.', 1),\n",
       " (u'guerre', 6),\n",
       " (u'malhabile', 1),\n",
       " (u\"look'st\", 4),\n",
       " (u'originated', 1),\n",
       " (u'calmer', 5),\n",
       " (u'flaxen', 1),\n",
       " (u\"minute's\", 1),\n",
       " (u'RODERIGO.--Bah!', 1),\n",
       " (u'twenty-two', 4),\n",
       " (u'material.', 2),\n",
       " (u'STOLZ,', 1),\n",
       " (u'Clouds', 2),\n",
       " (u'KNOWLTON', 1),\n",
       " (u'Hardly', 12),\n",
       " (u'Bourbon', 1),\n",
       " (u'granted?...\"', 1),\n",
       " (u'you--the', 1),\n",
       " (u'demande.', 2),\n",
       " (u'finish.', 10),\n",
       " (u\"Beauharnais'\", 2),\n",
       " (u'Abramovna', 1),\n",
       " (u'rings', 8),\n",
       " (u'like...', 2),\n",
       " (u'BUT', 2),\n",
       " (u'known...\"', 1),\n",
       " (u'd\\xe9serte:', 1),\n",
       " (u'sais', 19),\n",
       " (u'distaste,', 1),\n",
       " (u'disprove', 1),\n",
       " (u'caught;', 1),\n",
       " (u'imprisonment,', 2),\n",
       " (u'sorts', 12),\n",
       " (u'muzzle,', 1),\n",
       " (u'Brumaire,', 1),\n",
       " (u\"'pon\", 1),\n",
       " (u'foe,', 9),\n",
       " (u'preserve', 17),\n",
       " (u'externally', 3),\n",
       " (u'noirceur.', 2),\n",
       " (u'loves', 53),\n",
       " (u'swindler!', 1),\n",
       " (u'rolled', 22),\n",
       " (u'pontoon', 1),\n",
       " (u'torrent,', 1),\n",
       " (u'distinctness.', 1),\n",
       " (u'sleepers.', 1),\n",
       " (u'red-nosed', 4),\n",
       " (u\"s'unirent\", 1),\n",
       " (u'alteration,', 2),\n",
       " (u'Evades', 1),\n",
       " (u'god-den.', 2),\n",
       " (u\"'and\", 2),\n",
       " (u'Countess,\"', 6),\n",
       " (u'distrust--but', 1),\n",
       " (u'Here!', 1),\n",
       " (u'dishonor', 2),\n",
       " (u'enemy?\"', 1),\n",
       " (u'you--', 3),\n",
       " (u'blasted', 10),\n",
       " (u'bras', 8),\n",
       " (u'foresight', 2),\n",
       " (u'Kuzmich--a', 1),\n",
       " (u'ring;', 1),\n",
       " (u'rum.', 1),\n",
       " (u'veer', 3),\n",
       " (u'Testament', 1),\n",
       " (u'BRUCE', 6),\n",
       " (u'coronation', 1),\n",
       " (u'disarms', 1),\n",
       " (u'wines,', 1),\n",
       " (u'Shore', 1),\n",
       " (u'against', 355),\n",
       " (u'Laura', 1),\n",
       " (u'sort;', 1),\n",
       " (u'Cassio', 103),\n",
       " (u'devotion', 23),\n",
       " (u'Convey', 1),\n",
       " (u'demander', 6),\n",
       " (u'votre-fille', 1),\n",
       " (u'adapted', 2),\n",
       " (u'Freemasons,', 1),\n",
       " (u'Whip', 4),\n",
       " (u'DESD\\xc9MONA.--Au', 1),\n",
       " (u'hilly.', 1),\n",
       " (u'\"Too', 3),\n",
       " (u'connection', 39),\n",
       " (u'vain', 14),\n",
       " (u'JAGO.--Soit.', 1),\n",
       " (u'addressing', 69),\n",
       " (u'Tushin.', 10),\n",
       " (u'offerings', 1),\n",
       " (u'\"intended.\"', 1),\n",
       " (u'Shout', 1),\n",
       " (u'dehors.)', 1),\n",
       " (u\"match'd,\", 2),\n",
       " (u'wider', 3),\n",
       " (u'honneurs', 1),\n",
       " (u'humming', 7),\n",
       " (u'politesse.', 1),\n",
       " (u'durst,', 1),\n",
       " (u'Galactic', 4),\n",
       " (u'weeps,', 1),\n",
       " (u'deviations', 3),\n",
       " (u'Petersbourg,', 1),\n",
       " (u'asks', 16),\n",
       " (u'demesnes', 1),\n",
       " (u're\\xe7u...', 1),\n",
       " (u'contemplate.', 1),\n",
       " (u'received--assumed', 1),\n",
       " (u'Early', 6),\n",
       " (u'devil...', 1),\n",
       " (u'flank?\"', 2),\n",
       " (u'lingo.\"', 1),\n",
       " (u'obedient', 3),\n",
       " (u'mane,', 2),\n",
       " (u'appears:', 1),\n",
       " (u'duties', 23),\n",
       " (u'disseat', 1),\n",
       " (u'friends,\"', 4),\n",
       " (u'ponds,', 1),\n",
       " (u'aies', 1),\n",
       " (u'man\"', 1),\n",
       " (u'drugs', 4),\n",
       " (u'Dmitri,', 4),\n",
       " (u'afterwards!\"', 1),\n",
       " (u'needed.', 7),\n",
       " (u'Traitors', 1),\n",
       " (u\"Revisit'st\", 1),\n",
       " (u'ramassez', 1),\n",
       " (u'seminarists,', 1),\n",
       " (u'dashing,', 2),\n",
       " (u\"Mercutio's\", 5),\n",
       " (u'patronizing', 4),\n",
       " (u'shambles_,', 1),\n",
       " (u'\"Women,\"', 1),\n",
       " (u'ordnance', 3),\n",
       " (u'Tickle', 1),\n",
       " (u'exception', 14),\n",
       " (u'pitied....\"', 1),\n",
       " (u'bridges.', 2),\n",
       " (u'disgraced,', 1),\n",
       " (u'gods.', 2),\n",
       " (u'rejoinder.', 2),\n",
       " (u\"d'elle.\", 3),\n",
       " (u'men?\"', 1),\n",
       " (u'revolutions,', 3),\n",
       " (u'Ce', 20),\n",
       " (u'aim', 96),\n",
       " (u'rests', 16),\n",
       " (u'misconstrued', 1),\n",
       " (u'divinities?', 1),\n",
       " (u'brotherly', 7),\n",
       " (u'advice,\"', 1),\n",
       " (u'quick!', 3),\n",
       " (u'locality.', 2),\n",
       " (u'chilling', 2),\n",
       " (u'l\\xe9g\\xe8res.', 2),\n",
       " (u'faith.', 5),\n",
       " (u\"'Fear\", 2),\n",
       " (u'policeman,', 2),\n",
       " (u\"learned's\", 1),\n",
       " (u'losing', 43),\n",
       " (u'jutting', 1),\n",
       " (u'consent,', 7),\n",
       " (u'onze', 1),\n",
       " (u'us--doing', 1),\n",
       " (u\"bailiff's\", 1),\n",
       " (u'tugged', 5),\n",
       " (u'DESD\\xc9MONA.--Quoi!', 2),\n",
       " (u'erred,', 1),\n",
       " (u'Pompey?', 1),\n",
       " (u'voyage.', 1),\n",
       " (u'top!', 1),\n",
       " (u'grown', 81),\n",
       " (u'quand', 49),\n",
       " (u'Slav', 1),\n",
       " (u'jolting', 4),\n",
       " (u'CVIII.', 1),\n",
       " (u'thunderbolts;', 1),\n",
       " (u'finger,', 8),\n",
       " (u'hated,', 5),\n",
       " (u'forehead', 32),\n",
       " (u'grapple', 1),\n",
       " (u'COLE,', 1),\n",
       " (u'\"Granddad.\"', 2),\n",
       " (u'Petersburg.', 36),\n",
       " (u'naive', 22),\n",
       " (u'speaks:', 5),\n",
       " (u'lunes', 1),\n",
       " (u'qualities)', 1),\n",
       " (u'impotence.', 1),\n",
       " (u'Suspended', 1),\n",
       " (u'ill-usage', 1),\n",
       " (u'MACDUFF,', 7),\n",
       " (u'posed', 1),\n",
       " (u'men!...', 1),\n",
       " (u'reasons.\"', 1),\n",
       " (u'qualifications', 1),\n",
       " (u'evil....', 1),\n",
       " (u'aides-de-camp,', 3),\n",
       " (u'Men,', 2),\n",
       " (u'dimness', 1),\n",
       " (u'concourse', 2),\n",
       " (u'Carrying,', 1),\n",
       " (u'required.', 2),\n",
       " (u'vieux', 4),\n",
       " (u'workings', 2),\n",
       " (u'labor--idleness--was', 1),\n",
       " (u'\"Madame', 1),\n",
       " (u'talking,', 14),\n",
       " (u'made.', 12),\n",
       " (u'peaked', 3),\n",
       " (u'shortly', 8),\n",
       " (u'recognizable.', 1),\n",
       " (u'thy', 741),\n",
       " (u'Gratiano._)', 1),\n",
       " (u'because...', 11),\n",
       " (u'FRIGATE.', 1),\n",
       " (u'bottle;', 1),\n",
       " (u'\"Drain', 1),\n",
       " (u'conseils', 1),\n",
       " (u'Confound', 2),\n",
       " (u'skillful', 12),\n",
       " (u'stillness', 11),\n",
       " (u'grow:', 1),\n",
       " (u'qualities', 10),\n",
       " (u'cool,', 1),\n",
       " (u'mason,', 2),\n",
       " (u\"unpress'd\", 1),\n",
       " (u'meadows...', 1),\n",
       " (u'veteran', 2),\n",
       " (u'Reassured,', 1),\n",
       " (u'soldats,', 1),\n",
       " (u'recevoir.', 1),\n",
       " (u'song,', 8),\n",
       " (u'burn,', 4),\n",
       " ...]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCounts = textFile.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)\n",
    "wordCounts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Cache\n",
    "Pull a data sets into a cluster-wide in-memory cache. \n",
    "\n",
    "Very useful when using iterative algorithms.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "data/books-eng/hamlet.txt MapPartitionsRDD[1] at textFile at NativeMethodAccessorImpl.java:0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find words that start with an 'H' or 'h'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u\"host's\", 2),\n",
       " (u'humid', 1),\n",
       " (u'huge,', 8),\n",
       " (u'hillock.', 1),\n",
       " (u'hurrah!', 4),\n",
       " (u'hurry?\"', 1),\n",
       " (u'Had', 59),\n",
       " (u'habilet\\xe9;', 1),\n",
       " (u'hibernation', 1),\n",
       " (u'heirs', 1),\n",
       " (u'harness,', 5),\n",
       " (u'historians--those', 1),\n",
       " (u'health...\"', 1),\n",
       " (u'hear,\"', 1),\n",
       " (u'honesty,', 3),\n",
       " (u'historians--the', 1),\n",
       " (u'himself.)', 1),\n",
       " (u'hands,', 71),\n",
       " (u'heels;', 1),\n",
       " (u'hopeful', 3),\n",
       " (u'her,\"', 12),\n",
       " (u'haste;', 4),\n",
       " (u'haquen\\xe9es', 1),\n",
       " (u'hampered', 2),\n",
       " (u\"ha't\", 1),\n",
       " (u'highroad--polished', 1),\n",
       " (u'host;', 1),\n",
       " (u'HUSTON,', 2),\n",
       " (u'hope', 101),\n",
       " (u'happened,', 17),\n",
       " (u'Ha,', 14),\n",
       " (u'HANSON,', 1),\n",
       " (u'Helena.', 1),\n",
       " (u'history.', 15),\n",
       " (u'HUM.', 1),\n",
       " (u'herbs,', 3),\n",
       " (u'home?\"', 3),\n",
       " (u'hut--and', 1),\n",
       " (u'hedge', 2),\n",
       " (u'him--in', 3),\n",
       " (u'http://pglaf.org/fundraising.', 2),\n",
       " (u'howling.', 2),\n",
       " (u'hastes', 1),\n",
       " (u'hugged', 5),\n",
       " (u'houses,', 21),\n",
       " (u'horizons', 1),\n",
       " (u'hosts', 1),\n",
       " (u'hate;', 2),\n",
       " (u\"hack'd\", 1),\n",
       " (u'harness....\"', 1),\n",
       " (u'hems.', 1),\n",
       " (u'husband', 97),\n",
       " (u'hour!', 2),\n",
       " (u'Hmph.', 1),\n",
       " (u'hounds:', 1),\n",
       " (u'hangings.', 1),\n",
       " (u'hovers.', 1),\n",
       " (u'handing', 13),\n",
       " (u\"headquarters.'\", 1),\n",
       " (u'Him!...', 1),\n",
       " (u'hors', 5),\n",
       " (u'hive', 13),\n",
       " (u'has:', 1),\n",
       " (u'house:', 6),\n",
       " (u'Hardly', 12),\n",
       " (u'hopelessly', 1),\n",
       " (u'holds,', 2),\n",
       " (u'haste-post-haste', 1),\n",
       " (u'him--felt', 1),\n",
       " (u'hide', 60),\n",
       " (u'her--and', 3),\n",
       " (u'hooked', 4),\n",
       " (u'heiresses', 4),\n",
       " (u'horrible.', 1),\n",
       " (u'health', 48),\n",
       " (u'horrors;', 1),\n",
       " (u'honours.', 1),\n",
       " (u'HOME,', 1),\n",
       " (u'hard!\"', 1),\n",
       " (u'hero', 17),\n",
       " (u'headquarters,', 3),\n",
       " (u'honest:', 1),\n",
       " (u'him--who', 1),\n",
       " (u'him--especially', 1),\n",
       " (u'hairs', 9),\n",
       " (u'Harder', 1),\n",
       " (u'hobbledehoy', 1),\n",
       " (u'hunter!', 2),\n",
       " (u'hectic', 1),\n",
       " (u\"hadn't\", 8),\n",
       " (u'Here!', 1),\n",
       " (u'happiness!', 5),\n",
       " (u'Hyperion', 1),\n",
       " (u'happening.', 4),\n",
       " (u'horses,\"', 2),\n",
       " (u'healths', 1),\n",
       " (u'Hohenlohe,', 1),\n",
       " (u'himself,', 163),\n",
       " (u'hinting', 1),\n",
       " (u'hearer,', 1),\n",
       " (u'homme!\"', 1),\n",
       " (u'hilts,', 1),\n",
       " (u'Hessian', 4),\n",
       " (u'Human', 2),\n",
       " (u'Hecuba!', 1),\n",
       " (u'households', 2),\n",
       " (u'him!\"', 31),\n",
       " (u\"honor--that's\", 1),\n",
       " (u'hommes.--Si', 1),\n",
       " (u'History,', 1),\n",
       " (u'head]', 1),\n",
       " (u'H\\xc9RAUT.', 1),\n",
       " (u'Hurry,', 2),\n",
       " (u'housewife', 4),\n",
       " (u'humming', 7),\n",
       " (u'happiness--floated', 1),\n",
       " (u'hide?', 1),\n",
       " (u'How,', 4),\n",
       " (u'hitting', 10),\n",
       " (u'Happily', 2),\n",
       " (u'hospital', 18),\n",
       " (u'ha-ve', 1),\n",
       " (u'hangs', 19),\n",
       " (u'humours!', 1),\n",
       " (u'headdress', 2),\n",
       " (u'higher,', 5),\n",
       " (u'hard,', 15),\n",
       " (u'high-pitched', 5),\n",
       " (u'happiness,\"', 1),\n",
       " (u'holiday?\"', 1),\n",
       " (u'hushed.', 1),\n",
       " (u'hut', 35),\n",
       " (u'husband.\"', 1),\n",
       " (u'holder', 2),\n",
       " (u'HANGAR.', 1),\n",
       " (u'Hollabrunn,', 1),\n",
       " (u'Hmm.', 4),\n",
       " (u'Han!', 6),\n",
       " (u\"hare's\", 3),\n",
       " (u'head;', 7),\n",
       " (u'holiday.', 1),\n",
       " (u'highly,', 3),\n",
       " (u'horsecloths,', 2),\n",
       " (u'Honneur', 2),\n",
       " (u\"haven't\", 33),\n",
       " (u'he,', 182),\n",
       " (u'heart-ache', 1),\n",
       " (u'haters.', 1),\n",
       " (u'hadst,', 1),\n",
       " (u'honneurs', 1),\n",
       " (u'Horatio:', 2),\n",
       " (u'horseflies', 1),\n",
       " (u'hull,', 3),\n",
       " (u'hot.\"', 2),\n",
       " (u'heavier.', 1),\n",
       " (u'hearts.', 5),\n",
       " (u'Holding', 7),\n",
       " (u'handle', 9),\n",
       " (u\"huntsman's\", 2),\n",
       " (u'happiness--ever', 1),\n",
       " (u'honey-and-nut', 1),\n",
       " (u'hilly.', 1),\n",
       " (u'him--evidently', 1),\n",
       " (u'her--might', 1),\n",
       " (u'Hurrah', 3),\n",
       " (u'Honneur;', 1),\n",
       " (u'Hoch', 1),\n",
       " (u'hated,', 5),\n",
       " (u'half-witted', 1),\n",
       " (u'hosts!', 1),\n",
       " (u'her--please', 1),\n",
       " (u\"hunt's-up\", 1),\n",
       " (u'HENSON,', 1),\n",
       " (u'heures', 5),\n",
       " (u'HORATIO,', 7),\n",
       " (u'heads.\"', 1),\n",
       " (u'he!\"', 1),\n",
       " (u'hideuse', 1),\n",
       " (u'himself),', 2),\n",
       " (u'horsecloth,', 1),\n",
       " (u'Hello,', 3),\n",
       " (u'hunts,', 3),\n",
       " (u'hereupon', 1),\n",
       " (u'hear!', 2),\n",
       " (u'Hey...', 1),\n",
       " (u'Hung', 1),\n",
       " (u\"hoof's\", 1),\n",
       " (u'hearer:', 1),\n",
       " (u'Heights.', 2),\n",
       " (u'hommes.', 1),\n",
       " (u'housewifery,', 1),\n",
       " (u'heir;', 2),\n",
       " (u'holes', 4),\n",
       " (u'happiness--the', 1),\n",
       " (u'him--he', 7),\n",
       " (u'handrails.', 1),\n",
       " (u'heure;', 1),\n",
       " (u'habitera', 1),\n",
       " (u'hates', 2),\n",
       " (u'habit.', 3),\n",
       " (u'hast:', 1),\n",
       " (u'help', 188),\n",
       " (u'heavily', 26),\n",
       " (u'highmost', 2),\n",
       " (u'Hmmm?', 1),\n",
       " (u'history--civil', 1),\n",
       " (u'Han...', 2),\n",
       " (u'hath', 297),\n",
       " (u'heures,', 1),\n",
       " (u'Hills--even', 1),\n",
       " (u'hits.', 1),\n",
       " (u'Has', 9),\n",
       " (u'householders', 1),\n",
       " (u'hardhearted', 2),\n",
       " (u'happiness.', 28),\n",
       " (u'His', 422),\n",
       " (u'Hadst', 4),\n",
       " (u'here...certainly', 1),\n",
       " (u'haste:', 1),\n",
       " (u'honesty?', 2),\n",
       " (u'hell!', 4),\n",
       " (u'horrible.\"', 1),\n",
       " (u'helps', 11),\n",
       " (u\"hush'd\", 1),\n",
       " (u'HOOTKINS', 1),\n",
       " (u'host:', 1),\n",
       " (u'he.--Villain,', 1),\n",
       " (u'Hetzelsdorf', 1),\n",
       " (u'hautboys', 1),\n",
       " (u'halt', 13),\n",
       " (u'hm?', 1),\n",
       " (u'harms.', 1),\n",
       " (u'heathenish', 1),\n",
       " (u'here...', 5),\n",
       " (u'himself--not', 1),\n",
       " (u'hat.', 4),\n",
       " (u'horreurs,', 1),\n",
       " (u'Heights', 3),\n",
       " (u'Hamlet!', 6),\n",
       " (u'homestead', 8),\n",
       " (u'happen!\"', 2),\n",
       " (u'hales,', 1),\n",
       " (u'heightened', 4),\n",
       " (u'had.', 5),\n",
       " (u'homme!', 1),\n",
       " (u'hangs,', 1),\n",
       " (u\"ha'\", 6),\n",
       " (u'helpless,', 1),\n",
       " (u'hurrah!\"', 3),\n",
       " (u'homme?', 3),\n",
       " (u'heading', 15),\n",
       " (u'hop,', 1),\n",
       " (u'husband--God', 1),\n",
       " (u'habit', 32),\n",
       " (u'Hark,', 6),\n",
       " (u'Hollandais', 2),\n",
       " (u'headquarters--because,', 1),\n",
       " (u'HODENFIELD,', 1),\n",
       " (u'honest-hearted', 1),\n",
       " (u'harder,', 3),\n",
       " (u'has;', 2),\n",
       " (u'house;', 9),\n",
       " (u\"Hamlet's\", 8),\n",
       " (u'helped?\"', 1),\n",
       " (u'Had,', 1),\n",
       " (u'harmlessly.', 1),\n",
       " (u'humiliate', 2),\n",
       " (u'holder,', 2),\n",
       " (u'harvested', 1),\n",
       " (u'healthy,', 3),\n",
       " (u'Ho!', 5),\n",
       " (u'hard.\"', 2),\n",
       " (u'hussars?\"', 2),\n",
       " (u'Hello?', 1),\n",
       " (u'hommes!--Dis-moi,', 1),\n",
       " (u'hollering', 2),\n",
       " (u'hearted,', 1),\n",
       " (u'houses?', 2),\n",
       " (u'HOLMAN', 1),\n",
       " (u'hardship', 1),\n",
       " (u'honest;', 2),\n",
       " (u'hate:', 1),\n",
       " (u\"Hasn't\", 2),\n",
       " (u'hurrying,', 1),\n",
       " (u'HAMLET', 367),\n",
       " (u'houses', 33),\n",
       " (u'herring:', 1),\n",
       " (u'horseflesh', 4),\n",
       " (u'Hot.', 1),\n",
       " (u'hundreds', 28),\n",
       " (u'Honor', 5),\n",
       " (u'handiwork.', 3),\n",
       " (u'horses.', 27),\n",
       " (u'hilts;', 1),\n",
       " (u'heave,', 1),\n",
       " (u'her--considerate,', 1),\n",
       " (u'hatches', 1),\n",
       " (u'happening!', 1),\n",
       " (u'hot!\"', 2),\n",
       " (u\"have't!\", 1),\n",
       " (u'howbeit', 2),\n",
       " (u'humour.', 1),\n",
       " (u'hunger', 12),\n",
       " (u'happiness....\"', 1),\n",
       " (u'have?\"', 3),\n",
       " (u'honor;', 1),\n",
       " (u'Huge', 2),\n",
       " (u'honest;--so', 1),\n",
       " (u'Heaven.\"', 1),\n",
       " (u'head,', 117),\n",
       " (u'hollow', 27),\n",
       " (u'honneur.', 1),\n",
       " (u'him--we', 1),\n",
       " (u'honor)', 1),\n",
       " (u'husbanded?', 1),\n",
       " (u\"highness'\", 5),\n",
       " (u'hmmm?', 1),\n",
       " (u'home?', 4),\n",
       " (u'hollows', 5),\n",
       " (u'habiles', 1),\n",
       " (u\"hear't\", 1),\n",
       " (u'huh,', 1),\n",
       " (u'Henry', 1),\n",
       " (u'handkerchief,', 11),\n",
       " (u'hideous?\"', 1),\n",
       " (u'hug', 3),\n",
       " (u'hunters.', 1),\n",
       " (u'horseback--raised', 1),\n",
       " (u'Huttesse', 1),\n",
       " (u'Hands', 1),\n",
       " (u'hoodman-blind?', 1),\n",
       " (u'hands?', 1),\n",
       " (u\"heap'd\", 3),\n",
       " (u'harmful--gave', 1),\n",
       " (u'hard;', 3),\n",
       " (u'hundred,\"', 1),\n",
       " (u'herein;', 1),\n",
       " (u'handsome', 90),\n",
       " (u'holiday', 6),\n",
       " (u'HUMAN:', 4),\n",
       " (u'he?', 25),\n",
       " (u'head:', 7),\n",
       " (u'humanity', 27),\n",
       " (u'hazard.', 2),\n",
       " (u'hasard', 1),\n",
       " (u'hot,', 5),\n",
       " (u'history', 106),\n",
       " (u'heart', 229),\n",
       " (u'honn\\xeate?', 2),\n",
       " (u'hitting,', 1),\n",
       " (u'heads.', 9),\n",
       " (u'housemaid.', 1),\n",
       " (u'howling', 4),\n",
       " (u'hand,\"', 3),\n",
       " (u'high,', 22),\n",
       " (u'Horatio;', 1),\n",
       " (u'highroad.', 4),\n",
       " (u'hostess.', 8),\n",
       " (u'hairy,', 2),\n",
       " (u'her--that', 2),\n",
       " (u'hark,', 1),\n",
       " (u'heart!', 5),\n",
       " (u'honors', 9),\n",
       " (u'historians.', 1),\n",
       " (u'how!', 1),\n",
       " (u'health?\"', 2),\n",
       " (u\"heart's\", 14),\n",
       " (u'heritage', 1),\n",
       " (u'horse?\"', 1),\n",
       " (u'howl.', 4),\n",
       " (u'herself,', 58),\n",
       " (u'hallway', 13),\n",
       " (u'hands--those', 1),\n",
       " (u\"H\\xc9RAUT.--C'est\", 1),\n",
       " (u'heel,', 2),\n",
       " (u'holy,', 1),\n",
       " (u'him--', 1),\n",
       " (u'hostelry,', 1),\n",
       " (u'here\"', 1),\n",
       " (u'holily;', 1),\n",
       " (u'happier.', 3),\n",
       " (u'Henker', 1),\n",
       " (u'history,\"', 2),\n",
       " (u'honeying', 1),\n",
       " (u'Herein', 2),\n",
       " (u\"happ'd.\", 1),\n",
       " (u'happiest,', 1),\n",
       " (u'hollow-cheeked', 1),\n",
       " (u'himself--I', 1),\n",
       " (u'humeurs.', 1),\n",
       " (u'heaps', 1),\n",
       " (u'Hide', 2),\n",
       " (u'Heart,', 2),\n",
       " (u'hero--but', 1),\n",
       " (u'hence?', 2),\n",
       " (u'Hercules,', 2),\n",
       " (u'hers?', 1),\n",
       " (u\"hiss'd\", 1),\n",
       " (u'HOLT,', 2),\n",
       " (u'Hmmm.', 1),\n",
       " (u'horseman,', 1),\n",
       " (u'hand:', 18),\n",
       " (u'Hero', 1),\n",
       " (u'heights,', 1),\n",
       " (u'handkerchiefs;', 1),\n",
       " (u'hereabouts', 1),\n",
       " (u'hither;', 2),\n",
       " (u'herd,', 1),\n",
       " (u'Hippolyte.', 2),\n",
       " (u'harnessing', 1),\n",
       " (u'hap', 4),\n",
       " (u'Hamburg,', 1),\n",
       " (u'hail!', 3),\n",
       " (u'harp,', 1),\n",
       " (u'Harvest', 1),\n",
       " (u'him...', 13),\n",
       " (u'happened.', 17),\n",
       " (u'horn,', 3),\n",
       " (u'haut', 1),\n",
       " (u'hunter', 13),\n",
       " (u'Hamlet,', 25),\n",
       " (u'headquarters?\"', 1),\n",
       " (u'Hollow.', 1),\n",
       " (u'hurting', 3),\n",
       " (u'here.\"', 9),\n",
       " (u'hotshot', 1),\n",
       " (u'hat!', 1),\n",
       " (u'hunted', 4),\n",
       " (u'holiday:', 1),\n",
       " (u'heeded.', 1),\n",
       " (u'hope;', 1),\n",
       " (u'howling,', 1),\n",
       " (u'had!', 2),\n",
       " (u'houses--it', 1),\n",
       " (u'HECATE', 4),\n",
       " (u'honourably.', 1),\n",
       " (u'Hurry.', 1),\n",
       " (u'halfway.', 3),\n",
       " (u'herald', 5),\n",
       " (u'hereafter:', 1),\n",
       " (u'homing', 1),\n",
       " (u'hol\\xe0,', 2),\n",
       " (u'horseflesh,', 1),\n",
       " (u'halfworld', 1),\n",
       " (u\"hate'\", 3),\n",
       " (u'hems,', 1),\n",
       " (u'hour....', 1),\n",
       " (u'harder', 8),\n",
       " (u'hare', 18),\n",
       " (u'house.', 82),\n",
       " (u'hush', 3),\n",
       " (u'Hart,', 2),\n",
       " (u'horned', 2),\n",
       " (u'heretic,', 1),\n",
       " (u'hives;', 1),\n",
       " (u'happening--Kutuzov', 1),\n",
       " (u'happens', 44),\n",
       " (u'heartily;', 1),\n",
       " (u'horn--played', 1),\n",
       " (u'home...', 1),\n",
       " (u\"hair'?\", 1),\n",
       " (u'here,\"', 30),\n",
       " (u'hydraulic', 3),\n",
       " (u'h\\xe9riss\\xe9es', 1),\n",
       " (u'house...', 3),\n",
       " (u'Hoth,', 1),\n",
       " (u'Helene--having', 1),\n",
       " (u'honor!\"', 9),\n",
       " (u'hinder', 12),\n",
       " (u'happen,\"', 4),\n",
       " (u'huntsmen', 10),\n",
       " (u'harsh.', 2),\n",
       " (u'horreur', 1),\n",
       " (u\"Helena.'\", 1),\n",
       " (u'hold!\"', 1),\n",
       " (u'historians', 95),\n",
       " (u'Hey?\"', 1),\n",
       " (u'Hit', 1),\n",
       " (u'himself!...', 1),\n",
       " (u'he.', 118),\n",
       " (u'half-severed', 1),\n",
       " (u'halted', 14),\n",
       " (u'happening,', 6),\n",
       " (u'Hark', 6),\n",
       " (u'Hungarian', 2),\n",
       " (u'him...I', 1),\n",
       " (u'his,', 13),\n",
       " (u'help:', 2),\n",
       " (u'himself.', 150),\n",
       " (u'Hail', 2),\n",
       " (u'H.', 4),\n",
       " (u'heel', 8),\n",
       " (u'Harness,', 1),\n",
       " (u'here!', 31),\n",
       " (u'Happiness', 1),\n",
       " (u'hostile,', 2),\n",
       " (u'heedful', 1),\n",
       " (u'have;', 4),\n",
       " (u'horse?', 1),\n",
       " (u'Henceforth', 2),\n",
       " (u\"How'd\", 1),\n",
       " (u'hurdle', 1),\n",
       " (u'Hist!', 1),\n",
       " (u'Hail!', 3),\n",
       " (u'highly.', 2),\n",
       " (u\"he'd\", 4),\n",
       " (u'Hotel.', 1),\n",
       " (u'H-a-a-a...', 1),\n",
       " (u'her', 4543),\n",
       " (u'hit--a', 1),\n",
       " (u'hazards', 1),\n",
       " (u'heroism', 7),\n",
       " (u'hothouses.', 1),\n",
       " (u'handkerchief!', 4),\n",
       " (u'Hoist', 1),\n",
       " (u'Hofkriegsrath', 6),\n",
       " (u'HAGANS', 1),\n",
       " (u'humour?', 1),\n",
       " (u'hardly', 81),\n",
       " (u'husband)', 1),\n",
       " (u'headache,', 1),\n",
       " (u'head', 515),\n",
       " (u'hard.', 2),\n",
       " (u\"hawk'd\", 1),\n",
       " (u'http://pglaf.org', 4),\n",
       " (u'hallooing--and', 1),\n",
       " (u'Hence-banished', 1),\n",
       " (u\"hussar's\", 4),\n",
       " (u'HARRISON', 1),\n",
       " (u\"hussars'\", 1),\n",
       " (u'her.\"', 21),\n",
       " (u'hole', 13),\n",
       " (u'herself', 239),\n",
       " (u'hands.', 60),\n",
       " (u'harm.\"', 1),\n",
       " (u'hesitant', 1),\n",
       " (u'hypocrites;', 1),\n",
       " (u'home.', 38),\n",
       " (u'harbour?', 1),\n",
       " (u'horses;', 6),\n",
       " (u'hovering', 1),\n",
       " (u'heiress', 7),\n",
       " (u'h\\xe9ritier', 1),\n",
       " (u'horses)', 1),\n",
       " (u'hug.', 5),\n",
       " (u'her?--The', 1),\n",
       " (u'heard:', 2),\n",
       " (u'hearts,', 11),\n",
       " (u'hours?', 3),\n",
       " (u'her...immediately!', 1),\n",
       " (u'household', 32),\n",
       " (u'ha-ha-ha.', 1),\n",
       " (u'hermit.', 1),\n",
       " (u'horrors', 10),\n",
       " (u'husband;', 3),\n",
       " (u'honn\\xeate.', 2),\n",
       " (u'hides', 5),\n",
       " (u'halt:', 1),\n",
       " (u'hautes', 1),\n",
       " (u'hoar', 4),\n",
       " (u'human-cyborg', 2),\n",
       " (u'Hope', 1),\n",
       " (u'horsecloths.', 1),\n",
       " (u'himself--who', 3),\n",
       " (u'HARTNEY', 1),\n",
       " (u'him?...', 1),\n",
       " (u'high-coloured.', 1),\n",
       " (u'hologram.', 1),\n",
       " (u'hallway.', 11),\n",
       " (u'hatch,', 1),\n",
       " (u'horror', 31),\n",
       " (u'heraldry,', 1),\n",
       " (u'handled.', 1),\n",
       " (u\"horse's\", 15),\n",
       " (u'higher.', 2),\n",
       " (u'had.\"', 1),\n",
       " (u'husbands', 12),\n",
       " (u'hitched', 1),\n",
       " (u'Haste', 2),\n",
       " (u'horreurs', 1),\n",
       " (u\"husband's\", 25),\n",
       " (u'happier', 10),\n",
       " (u'happier?', 1),\n",
       " (u\"head's\", 3),\n",
       " (u'Hutt.', 5),\n",
       " (u'house-affairs', 1),\n",
       " (u'historic,', 1),\n",
       " (u'humanity.', 7),\n",
       " (u'him--at', 1),\n",
       " (u'here!...\"', 1),\n",
       " (u'Holla!', 2),\n",
       " (u'humor,', 6),\n",
       " (u'housemaid...', 1),\n",
       " (u'hardi', 2),\n",
       " (u'high-battled', 1),\n",
       " (u'hands', 208),\n",
       " (u'hits:', 1),\n",
       " (u'handicraft,', 1),\n",
       " (u'hurlements', 1),\n",
       " (u'happen),', 1),\n",
       " (u'hastens', 2),\n",
       " (u'heiress,', 1),\n",
       " (u'hits,', 1),\n",
       " (u'hence,', 13),\n",
       " (u'happen--despite', 1),\n",
       " (u'hand;', 18),\n",
       " (u'heave.', 2),\n",
       " (u'haughty', 2),\n",
       " (u'homage', 5),\n",
       " (u'hums', 1),\n",
       " (u'headdress,', 1),\n",
       " (u\"Honour'd,\", 1),\n",
       " (u'he?!', 1),\n",
       " (u'histories', 15),\n",
       " (u'heureuse.', 1),\n",
       " (u\"hand'\", 1),\n",
       " (u'hither:', 6),\n",
       " (u'Hic', 1),\n",
       " (u'households,', 2),\n",
       " (u'hardened', 2),\n",
       " (u'History', 11),\n",
       " (u\"house's\", 1),\n",
       " (u'honte', 3),\n",
       " (u'Hamlet?', 4),\n",
       " (u'hawkers,', 3),\n",
       " (u'hostess,', 4),\n",
       " (u'horologe', 1),\n",
       " (u'HORATIO', 114),\n",
       " (u'houses!', 4),\n",
       " (u\"husband,'fall'st\", 1),\n",
       " (u'honorable', 13),\n",
       " (u'Historic', 1),\n",
       " (u'Hills;', 1),\n",
       " (u'him', 3435),\n",
       " (u'hovel', 1),\n",
       " (u'himself?\"...', 1),\n",
       " (u'harbinger', 1),\n",
       " (u'hereabout:', 2),\n",
       " (u'heaved', 7),\n",
       " (u'human.', 1),\n",
       " (u'hair?\"', 1),\n",
       " (u'hers,', 11),\n",
       " (u'Hyperdrive.', 1),\n",
       " (u\"Highness'\", 2),\n",
       " (u'hereafter;', 2),\n",
       " (u'hero,', 8),\n",
       " (u'Here!\"', 1),\n",
       " (u'hopelessness:', 1),\n",
       " (u'help;', 2),\n",
       " (u'horse),', 1),\n",
       " (u'here\"--and', 1),\n",
       " (u'hussar!\"', 1),\n",
       " (u'humbug,', 1),\n",
       " (u\"hail'd\", 1),\n",
       " (u'habitually', 1),\n",
       " (u'hard', 93),\n",
       " (u'hair-breadth', 1),\n",
       " (u'honneur,', 2),\n",
       " (u'healed', 2),\n",
       " (u'hut,', 14),\n",
       " (u'Hold;', 1),\n",
       " (u'hanged', 7),\n",
       " (u'hindrance', 7),\n",
       " (u'heels!\"', 1),\n",
       " (u'hurries', 12),\n",
       " (u'Hyrcan', 1),\n",
       " (u'humiliation.', 1),\n",
       " (u'hourly', 2),\n",
       " (u\"heaven's\", 20),\n",
       " (u'holder.', 2),\n",
       " (u'hussar,', 7),\n",
       " (u'hinds?', 1),\n",
       " (u'headquarters,\"', 1),\n",
       " (u'hilt', 3),\n",
       " (u'hearse,', 1),\n",
       " (u'honour!', 1),\n",
       " (u'howsoever', 1),\n",
       " (u'hell-hound,', 1),\n",
       " (u'hayfield.', 1),\n",
       " (u'harvesting,\"', 1),\n",
       " (u\"ha't,\", 1),\n",
       " (u'Happy', 4),\n",
       " (u'half-closed', 4),\n",
       " (u'hobby', 1),\n",
       " (u'Heaven', 20),\n",
       " (u'heureux', 2),\n",
       " (u'he...\"', 1),\n",
       " (u\"honor'\", 1),\n",
       " (u'Hokey', 1),\n",
       " (u'Hover', 1),\n",
       " (u'healing', 2),\n",
       " (u'heroic.', 1),\n",
       " (u'Hot,', 1),\n",
       " (u'hunters,', 1),\n",
       " (u'hostile', 18),\n",
       " (u'hunger-stricken,', 1),\n",
       " (u'hearsed', 1),\n",
       " (u'humbugged', 2),\n",
       " (u'hovers', 3),\n",
       " (u'hid,', 2),\n",
       " (u'herself.', 41),\n",
       " (u'happiness,', 12),\n",
       " (u'heirs?\"', 1),\n",
       " (u'hooted', 1),\n",
       " (u'here-approach,', 1),\n",
       " (u'handleless', 1),\n",
       " (u'Hence,', 5),\n",
       " (u'Hungarian?\"', 1),\n",
       " (u'hotly', 3),\n",
       " (u'humblement', 4),\n",
       " (u\"How's\", 2),\n",
       " (u'heard;', 1),\n",
       " (u'hustles', 1),\n",
       " (u'have:', 4),\n",
       " (u'Hussars,', 2),\n",
       " (u'hitherto', 12),\n",
       " (u'hope:', 2),\n",
       " (u'How!', 7),\n",
       " (u\"hawk's\", 1),\n",
       " (u'heated', 7),\n",
       " (u'HOLLY', 1),\n",
       " (u'haine,', 2),\n",
       " (u'hawk', 2),\n",
       " (u'hunger,\"', 1),\n",
       " (u'ha\\xefsse', 1),\n",
       " (u'humour,', 4),\n",
       " (u'happened,\"', 5),\n",
       " (u'honor.\\'\"', 1),\n",
       " (u'Hussar,', 1),\n",
       " (u'HARRELL', 1),\n",
       " (u'hundred', 134),\n",
       " (u'halt;', 1),\n",
       " (u'happy', 176),\n",
       " (u'hesitations,', 1),\n",
       " (u\"hatch'd,\", 1),\n",
       " (u'himself,\"', 4),\n",
       " (u'heroics.', 1),\n",
       " (u'happiness!\"', 3),\n",
       " (u'Ha-ha-ha!', 1),\n",
       " (u'hands!', 1),\n",
       " (u'headset)', 21),\n",
       " (u'HOLOGRAM', 1),\n",
       " (u'holographic', 5),\n",
       " (u'hold', 171),\n",
       " (u'haven', 2),\n",
       " (u'he', 8766),\n",
       " (u'Hurry', 3),\n",
       " (u\"hell's\", 1),\n",
       " (u'How', 375),\n",
       " (u'hideously', 1),\n",
       " (u'horses:', 1),\n",
       " (u'honor?...\"', 1),\n",
       " (u'hazard,', 1),\n",
       " (u'hillside.', 2),\n",
       " (u'Helene', 88),\n",
       " (u\"here--that's\", 1),\n",
       " (u'headache', 3),\n",
       " (u'Himself', 3),\n",
       " (u'helm,', 1),\n",
       " (u'hair,', 36),\n",
       " (u'hideux', 1),\n",
       " (u'housekeeper)', 1),\n",
       " (u'history--whatever', 1),\n",
       " (u'hesitates', 1),\n",
       " (u'high-minded', 1),\n",
       " (u'HERALD,', 1),\n",
       " (u'Holds', 4),\n",
       " (u'hear?\"', 5),\n",
       " (u'heard', 480),\n",
       " (u'hours,', 15),\n",
       " (u'high.', 3),\n",
       " (u'husband:', 4),\n",
       " (u'honey,', 3),\n",
       " (u'HERMAN', 2),\n",
       " (u'historians:', 1),\n",
       " (u'hesitate', 1),\n",
       " (u'horses!\"', 1),\n",
       " (u\"Hendrikhovna's\", 5),\n",
       " (u'harrows', 1),\n",
       " (u'hardness,', 1),\n",
       " (u'hark.', 1),\n",
       " (u'house,--But,', 1),\n",
       " (u'heels?\"', 1),\n",
       " (u'haps,', 1),\n",
       " (u'Hospital,', 1),\n",
       " (u'historic', 34),\n",
       " (u'historians,', 13),\n",
       " (u'health...', 1),\n",
       " (u'hem!_', 1),\n",
       " (u'hither?\"', 1),\n",
       " (u\"He'll\", 15),\n",
       " (u'hurry?\"--Nicholas', 1),\n",
       " (u'howls.', 4),\n",
       " (u'hospital.', 5),\n",
       " (u\"hand's\", 2),\n",
       " (u'hadst', 13),\n",
       " (u'harbingers', 2),\n",
       " (u'habitual,', 1),\n",
       " (u'hazel', 3),\n",
       " (u'hearing;', 1),\n",
       " (u'highroad,', 3),\n",
       " (u'him--Rostov.', 1),\n",
       " (u'humanity.\"', 1),\n",
       " (u\"Hutt's\", 1),\n",
       " (u'Hum!', 4),\n",
       " (u'hare.', 7),\n",
       " (u'haltingly)', 1),\n",
       " (u'hoarsely', 3),\n",
       " (u'happies', 1),\n",
       " (u'honestly.', 1),\n",
       " (u'honorably,\"', 1),\n",
       " (u'heirs.', 1),\n",
       " (u'hard--', 1),\n",
       " (u'honor', 80),\n",
       " (u'hearty', 5),\n",
       " (u'him--Pierre--depriving', 1),\n",
       " (u'handgun.', 1),\n",
       " (u'howl,', 5),\n",
       " (u'helplessly', 6),\n",
       " (u'harm!_', 1),\n",
       " (u'here...are', 1),\n",
       " (u'Horse', 21),\n",
       " (u'hierarchy', 2),\n",
       " (u'haute', 5),\n",
       " (u'hardtack', 1),\n",
       " (u'HARMAN,', 1),\n",
       " (u'hence!', 4),\n",
       " (u'HUME,', 2),\n",
       " (u'humaine', 2),\n",
       " (u'headset.', 1),\n",
       " (u'herself,\"', 1),\n",
       " (u'happiness', 77),\n",
       " (u'hand--he', 1),\n",
       " (u'haven.', 1),\n",
       " (u'HUT', 1),\n",
       " (u'heavenly;', 1),\n",
       " (u'honoring', 1),\n",
       " (u'had', 5563),\n",
       " (u'hyperspace', 6),\n",
       " (u'hey-day', 1),\n",
       " (u'h\\xe9las!', 4),\n",
       " (u'Hid', 1),\n",
       " (u'heights.', 1),\n",
       " (u'herdsman', 1),\n",
       " (u'harsh,', 2),\n",
       " (u'him),', 2),\n",
       " (u'homepage', 1),\n",
       " (u\"hideux.--D'ailleurs\", 1),\n",
       " (u'house.]', 2),\n",
       " (u'hair!...\"', 1),\n",
       " (u'hundred?\"', 1),\n",
       " (u\"Han's\", 28),\n",
       " (u'hanged.', 1),\n",
       " (u'Hamlet.', 11),\n",
       " (u'hugging', 3),\n",
       " (u'huddles', 2),\n",
       " (u'harp.', 2),\n",
       " (u'Home\"', 2),\n",
       " (u'ha,', 17),\n",
       " (u'hiccough', 1),\n",
       " (u'hey!\"', 1),\n",
       " (u'Hurrah!', 3),\n",
       " (u'honor?...', 1),\n",
       " (u'hate!', 4),\n",
       " (u'hilding!', 1),\n",
       " (u'happily...\"', 1),\n",
       " (u'hanging,', 1),\n",
       " (u'Hate', 2),\n",
       " (u'heavy', 106),\n",
       " (u'hairy', 9),\n",
       " (u'HAROLD', 1),\n",
       " (u'hales', 1),\n",
       " (u'hurt,', 6),\n",
       " (u'humans', 1),\n",
       " (u'house,', 77),\n",
       " (u'her;', 33),\n",
       " (u'H\\xe9las!', 2),\n",
       " (u'heralds', 3),\n",
       " (u'Hurrah!...\"', 1),\n",
       " (u'humane', 2),\n",
       " (u'heave', 4),\n",
       " (u'hobby-horse:', 1),\n",
       " (u'hesitation', 11),\n",
       " (u'hall--\"but', 1),\n",
       " (u'heat-oppressed', 1),\n",
       " (u'HOWARTH', 1),\n",
       " (u'hit;', 1),\n",
       " (u'hearted;', 1),\n",
       " (u'HAMLET,', 4),\n",
       " (u'haunt,', 2),\n",
       " (u'he?\"', 11),\n",
       " (u'hirelings', 1),\n",
       " (u'honors.', 1),\n",
       " (u'Hoth.', 1),\n",
       " (u'him\"', 4),\n",
       " (u'hear?', 8),\n",
       " (u'habitude?', 1),\n",
       " (u'Hecate!', 1),\n",
       " (u'headphones.', 1),\n",
       " (u'heathen?', 1),\n",
       " (u'Hostilities', 1),\n",
       " (u'hey,', 1),\n",
       " (u'harmony!\"', 1),\n",
       " (u'horseflesh!\\'\"', 1),\n",
       " (u'hoarse', 17),\n",
       " (u'harshly', 2),\n",
       " (u'hurts,', 2),\n",
       " (u'handwriting', 1),\n",
       " (u'hait', 1),\n",
       " (u'here;', 16),\n",
       " (u'hits', 35),\n",
       " (u'his--has', 1),\n",
       " (u'Huttese.', 2),\n",
       " (u'his.', 13),\n",
       " (u'Hence!', 3),\n",
       " (u'heavy;', 1),\n",
       " (u'her--for', 1),\n",
       " (u'head)', 4),\n",
       " (u'hallway,', 8),\n",
       " (u'herself!', 4),\n",
       " (u'happiness,\\'\"', 1),\n",
       " (u'hams:', 1),\n",
       " (u\"hero's\", 1),\n",
       " (u'hummed', 3),\n",
       " (u'humour', 8),\n",
       " (u'hereditary,', 1),\n",
       " (u'habitual', 17),\n",
       " (u'her)', 2),\n",
       " (u'hard-boiled', 1),\n",
       " (u'hangers,', 1),\n",
       " (u'high.\"', 1),\n",
       " (u\"HAN'S\", 1),\n",
       " (u'h\\xe2ter', 1),\n",
       " (u'Herculean', 2),\n",
       " (u'horizontal', 2),\n",
       " (u'harm,\"', 2),\n",
       " (u'hangar,', 4),\n",
       " (u'highest,', 1),\n",
       " (u'horse', 162),\n",
       " (u'history),', 1),\n",
       " (u'horsemen.', 2),\n",
       " (u'hastening', 5),\n",
       " (u'Helene.', 17),\n",
       " (u'harder--ever', 1),\n",
       " (u'hellish', 3),\n",
       " (u\"huntsmen's\", 3),\n",
       " (u'hawking', 1),\n",
       " (u'hist!', 1),\n",
       " (u'honesty:', 1),\n",
       " (u'Horizontal', 1),\n",
       " (u'Herself', 2),\n",
       " (u'home,', 41),\n",
       " (u'held:', 1),\n",
       " (u'hypothetical', 1),\n",
       " (u'http://www.pglaf.org.', 2),\n",
       " (u'Himself,', 2),\n",
       " (u'histories,', 1),\n",
       " (u'huh?', 7),\n",
       " (u'hoarfrost.', 2),\n",
       " (u'hiding.', 2),\n",
       " (u'homely', 3),\n",
       " (u'heat', 30),\n",
       " (u'hoist', 2),\n",
       " (u'Hippolyte,', 8),\n",
       " (u'hateful', 8),\n",
       " (u'hopes', 27),\n",
       " (u'happened!\"', 1),\n",
       " (u'hindrance,', 1),\n",
       " (u\"here's\", 30),\n",
       " (u'heels--we', 1),\n",
       " (u'half,', 4),\n",
       " (u'happiness.\"', 3),\n",
       " (u'hay.', 2),\n",
       " (u'her....', 1),\n",
       " (u'he:', 5),\n",
       " (u\"HAMLET.'\", 2),\n",
       " (u'Helen.', 1),\n",
       " (u'hospitality,', 3),\n",
       " (u\"hate's\", 2),\n",
       " (u'harm--and', 1),\n",
       " (u'Husband?', 1),\n",
       " (u'harshly,\"', 1),\n",
       " (u'Hood', 1),\n",
       " (u'honn\\xeate,', 2),\n",
       " (u'huh!', 2),\n",
       " (u'high', 142),\n",
       " (u'http://www.gutenberg.org/1/8/1/7/18179/', 1),\n",
       " (u'him--also', 1),\n",
       " (u'have', 3081),\n",
       " (u'heaven,', 49),\n",
       " (u'hymns', 3),\n",
       " (u'Hermit,', 1),\n",
       " (u'he:--O', 1),\n",
       " (u'hell:', 2),\n",
       " (u'horns,', 1),\n",
       " (u'half-smile', 1),\n",
       " (u'h\\xe2tez-vous,', 1),\n",
       " (u'horse,\"', 4),\n",
       " (u'hat--a', 1),\n",
       " (u\"hour's\", 5),\n",
       " ...]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = textFile.flatMap(lambda line: line.split())\n",
    "words_subset = words.filter(lambda x: x[0] in ['H','h'])\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "counts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleanup using a regular expression\n",
    "\n",
    "* remove all non-alphanumeric characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all 110\n",
      "antic 1\n",
      "armour 3\n",
      "actions 2\n",
      "admittance 1\n",
      "aspect 1\n",
      "amiss 2\n",
      "appetite 1\n",
      "accurst 1\n",
      "Awake 1\n",
      "advanced 1\n",
      "admirable 1\n",
      "Away 2\n",
      "abused 1\n",
      "assay 4\n",
      "advantage 1\n",
      "account 1\n",
      "answer 14\n",
      "Anon 3\n",
      "accuse 1\n",
      "animals 1\n",
      "affections 1\n",
      "Affection 1\n",
      "Art 1\n",
      "arrows 2\n",
      "abatements 1\n",
      "audience 5\n",
      "argal 3\n",
      "arrow 1\n",
      "Attends 1\n",
      "Aroused 1\n",
      "antique 2\n",
      "achievements 1\n",
      "art 16\n",
      "afflict 1\n",
      "asleep 2\n",
      "aloof 2\n",
      "appurtenance 1\n",
      "Attendants 9\n",
      "attend 1\n",
      "armed 2\n",
      "away 23\n",
      "access 1\n",
      "acting 1\n",
      "above 4\n",
      "assurance 2\n",
      "ass 4\n",
      "accord 1\n",
      "assured 2\n",
      "ask 2\n",
      "Against 4\n",
      "abate 1\n",
      "Addicted 1\n",
      "along 3\n",
      "about 15\n",
      "abstinence 1\n",
      "accepts 1\n",
      "assault 1\n",
      "against 19\n",
      "aye 1\n",
      "anon 3\n",
      "Advancing 1\n",
      "action 10\n",
      "approve 2\n",
      "altitude 1\n",
      "amber 1\n",
      "acted 1\n",
      "apt 2\n",
      "arras 5\n",
      "alone 10\n",
      "alleys 1\n",
      "another 11\n",
      "Answer 1\n",
      "array 1\n",
      "afoot 1\n",
      "abominably 1\n",
      "acquaint 1\n",
      "addition 3\n",
      "angry 1\n",
      "accent 2\n",
      "audit 1\n",
      "avouch 1\n",
      "adoption 1\n",
      "angels 2\n",
      "a 469\n",
      "A 81\n",
      "Assume 1\n",
      "awake 1\n",
      "ashamed 1\n",
      "And 264\n",
      "admiration 3\n",
      "aught 10\n",
      "apprehension 2\n",
      "arrant 2\n",
      "appliance 1\n",
      "aim 2\n",
      "Almost 1\n",
      "assays 1\n",
      "amble 1\n",
      "age 9\n",
      "affection 3\n",
      "and 706\n",
      "angel 4\n",
      "almost 9\n",
      "abstract 1\n",
      "am 51\n",
      "affrighted 1\n",
      "anger 1\n",
      "as 154\n",
      "ay 3\n",
      "appointment 1\n",
      "awhile 12\n",
      "Angels 1\n",
      "apoplex 1\n",
      "angle 1\n",
      "agreeing 1\n",
      "able 1\n",
      "admit 1\n",
      "actor 3\n",
      "actors 2\n",
      "ambitious 3\n",
      "afflicts 1\n",
      "advise 1\n",
      "arithmetic 1\n",
      "ancle 1\n",
      "About 3\n",
      "allow 3\n",
      "accidental 1\n",
      "applaud 1\n",
      "advice 3\n",
      "apparel 1\n",
      "Admit 1\n",
      "arrived 1\n",
      "attent 1\n",
      "authorities 1\n",
      "awe 2\n",
      "ago 1\n",
      "All 12\n",
      "assume 3\n",
      "afar 1\n",
      "alarm 2\n",
      "Am 2\n",
      "As 75\n",
      "aunt 1\n",
      "Another 6\n",
      "Ay 35\n",
      "Armed 1\n",
      "avoid 3\n",
      "amazed 1\n",
      "affair 4\n",
      "After 2\n",
      "ambassador 1\n",
      "ambition 6\n",
      "Acts 1\n",
      "argues 1\n",
      "arrests 1\n",
      "aright 1\n",
      "adders 1\n",
      "anticipation 1\n",
      "Ambassador 1\n",
      "advancement 2\n",
      "acres 1\n",
      "amaze 1\n",
      "Alas 9\n",
      "attractive 1\n",
      "augury 1\n",
      "Affront 1\n",
      "appal 1\n",
      "assistant 2\n",
      "adheres 1\n",
      "auspicious 1\n",
      "Adam 2\n",
      "altogether 1\n",
      "artery 1\n",
      "adventurous 1\n",
      "apart 2\n",
      "articles 1\n",
      "always 1\n",
      "annexment 1\n",
      "Appears 1\n",
      "Are 8\n",
      "arraign 1\n",
      "Ambassadors 2\n",
      "At 12\n",
      "Arm 3\n",
      "amazement 2\n",
      "Aside 11\n",
      "actively 1\n",
      "are 123\n",
      "awry 1\n",
      "ambassadors 3\n",
      "afeard 1\n",
      "arm 4\n",
      "ability 1\n",
      "army 1\n",
      "Adieu 4\n",
      "appear 6\n",
      "asking 2\n",
      "across 1\n",
      "attended 2\n",
      "abridgement 1\n",
      "amen 1\n",
      "apparition 2\n",
      "answerest 1\n",
      "According 1\n",
      "ancient 1\n",
      "address 1\n",
      "ache 2\n",
      "article 2\n",
      "appears 2\n",
      "Among 1\n",
      "accident 3\n",
      "asunder 1\n",
      "annual 1\n",
      "according 1\n",
      "Antiquity 1\n",
      "assure 1\n",
      "abuse 1\n",
      "afternoon 1\n",
      "act 14\n",
      "asked 1\n",
      "allowance 2\n",
      "already 6\n",
      "afterwards 1\n",
      "Act 5\n",
      "allegiance 1\n",
      "abhorred 1\n",
      "affliction 3\n",
      "ape 2\n",
      "Aeneas 1\n",
      "arrest 1\n",
      "arms 9\n",
      "abroad 1\n",
      "airs 1\n",
      "ah 2\n",
      "airy 1\n",
      "astonish 1\n",
      "aslant 1\n",
      "acquire 1\n",
      "alas 1\n",
      "assail 1\n",
      "aery 1\n",
      "air 13\n",
      "apiece 1\n",
      "aid 1\n",
      "artless 1\n",
      "anchor 1\n",
      "Alexander 5\n",
      "attribute 1\n",
      "argument 4\n",
      "an 52\n",
      "associates 1\n",
      "at 75\n",
      "affectation 1\n",
      "any 17\n",
      "absolute 2\n",
      "again 32\n",
      "afraid 1\n",
      "author 2\n",
      "adjoin 1\n",
      "anoint 1\n",
      "acts 1\n",
      "aside 1\n",
      "aptly 1\n",
      "also 1\n",
      "adulterate 1\n",
      "aboard 3\n",
      "Absent 1\n",
      "adieu 3\n",
      "assigns 2\n",
      "affairs 1\n",
      "amities 1\n",
      "after 12\n",
      "acquaintance 1\n",
      "ardour 1\n",
      "Abuses 1\n",
      "Alack 3\n",
      "ambiguous 1\n",
      "Ah 4\n",
      "An 7\n",
      "accounted 1\n",
      "absurd 2\n",
      "axe 3\n",
      "accidents 1\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile('[\\W_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "\n",
    "#first_letters = set(['H','h','Q','q','s','S'])\n",
    "first_letters = set(['A','a'])\n",
    "# Count words that start with H\n",
    "words_subset = words.filter(lambda x: x[0] in first_letters)\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "res = counts.collect()\n",
    "for r,c in res:\n",
    "    print r,c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting Letter bigrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "#textFile = sc.textFile(\"notes/data/books-eng/hamlet.txt\")\n",
    "textFile = sc.textFile(\"data/books-eng\")\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile(u'[\\W0-9_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "# Convert to lower case\n",
    "words = words.map(lambda w: w.lower())\n",
    "\n",
    "# Convert to a list of letters list('abc') = ['a','b','c']\n",
    "letters = words.flatMap(lambda word: [pair[0]+pair[1] for pair in zip(list('_'+word),list(word+'_')) ] )\n",
    "counts = letters.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "bigrams = counts.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e_ 173007\n",
      "_t 128896\n",
      "th 106890\n",
      "s_ 102616\n",
      "he 101877\n",
      "d_ 99475\n",
      "_a 97158\n",
      "t_ 92487\n",
      "_s 70680\n",
      "_h 67773\n",
      "in 66235\n",
      "an 64815\n",
      "n_ 63820\n",
      "er 63564\n",
      "_w 56282\n",
      "r_ 53687\n",
      "re 51632\n",
      "_i 49719\n",
      "_o 48674\n",
      "y_ 47659\n",
      "nd 47369\n",
      "o_ 43800\n",
      "on 40854\n",
      "ou 40280\n",
      "en 40046\n",
      "at 39581\n",
      "ha 39353\n",
      "_b 38727\n",
      "_m 36324\n",
      "_c 35647\n",
      "hi 35341\n",
      "to 34894\n",
      "ed 33922\n",
      "is 33107\n",
      "ng 32960\n",
      "_f 32956\n",
      "es 32443\n",
      "_d 31614\n",
      "it 31281\n",
      "or 31094\n",
      "ar 30018\n",
      "g_ 28707\n",
      "_p 28563\n",
      "_l 28281\n",
      "as 28232\n",
      "st 28109\n",
      "te 27940\n",
      "le 27823\n",
      "f_ 26631\n",
      "se 26373\n",
      "nt 25097\n",
      "a_ 24603\n",
      "me 24098\n",
      "h_ 23958\n",
      "l_ 23453\n",
      "_n 23156\n",
      "ve 23023\n",
      "de 22741\n",
      "of 22605\n",
      "ll 22459\n",
      "ro 22383\n",
      "_r 21805\n",
      "ea 21744\n",
      "ne 21493\n",
      "al 21160\n",
      "_e 20821\n",
      "ti 19941\n",
      "ho 19929\n",
      "no 19729\n",
      "co 19284\n",
      "ri 19224\n",
      "ce 18776\n",
      "m_ 18059\n",
      "be 17505\n",
      "om 16780\n",
      "el 16726\n",
      "wh 16362\n",
      "ut 16327\n",
      "ch 16300\n",
      "_g 16170\n",
      "us 15654\n",
      "ot 15603\n",
      "ur 15515\n",
      "ma 15342\n",
      "wi 15264\n",
      "wa 15242\n",
      "sh 15099\n",
      "ad 14911\n",
      "ow 14882\n",
      "lo 14702\n",
      "li 14346\n",
      "et 14133\n",
      "ra 13998\n",
      "si 13911\n",
      "ee 13729\n",
      "ss 13609\n",
      "ai 13581\n",
      "il 13499\n",
      "so 13246\n",
      "ta 13229\n",
      "ie 13151\n",
      "_y 13113\n",
      "un 13042\n",
      "i_ 13034\n",
      "la 12952\n",
      "im 12935\n",
      "pe 12401\n",
      "fo 12390\n",
      "ly 12176\n",
      "rs 12118\n",
      "io 11992\n",
      "nc 11800\n",
      "w_ 11671\n",
      "ic 11600\n",
      "oo 11477\n",
      "yo 11459\n",
      "we 11425\n",
      "ca 11390\n",
      "sa 11272\n",
      "id 11247\n",
      "di 11239\n",
      "mo 11017\n",
      "ac 10786\n",
      "ol 10729\n",
      "pr 10559\n",
      "tr 10317\n",
      "em 10279\n",
      "ke 10143\n",
      "u_ 10091\n",
      "ir 9822\n",
      "ns 9769\n",
      "ld 9755\n",
      "do 9531\n",
      "os 9357\n",
      "ge 9287\n",
      "ul 9201\n",
      "gh 9156\n",
      "ov 8964\n",
      "_u 8948\n",
      "k_ 8845\n",
      "rt 8572\n",
      "ni 8533\n",
      "ay 8478\n",
      "_v 7962\n",
      "pa 7962\n",
      "mi 7841\n",
      "av 7793\n",
      "na 7772\n",
      "po 7752\n",
      "ec 7708\n",
      "fr 7596\n",
      "bu 7534\n",
      "am 7441\n",
      "ig 7416\n",
      "ry 7187\n",
      "rd 7177\n",
      "fi 7118\n",
      "ev 7090\n",
      "wo 7065\n",
      "vi 6727\n",
      "fe 6568\n",
      "ht 6539\n",
      "pi 6433\n",
      "fa 6346\n",
      "rr 6332\n",
      "tt 6262\n",
      "ts 6242\n",
      "go 6233\n",
      "su 6067\n",
      "pl 6044\n",
      "bo 5934\n",
      "_k 5928\n",
      "ag 5854\n",
      "ct 5848\n",
      "ap 5810\n",
      "tu 5809\n",
      "ei 5797\n",
      "ia 5646\n",
      "bl 5635\n",
      "ba 5542\n",
      "qu 5413\n",
      "ey 5348\n",
      "p_ 5299\n",
      "if 5295\n",
      "ep 5285\n",
      "my 5275\n",
      "sp 5248\n",
      "od 5187\n",
      "mp 5122\n",
      "iv 5121\n",
      "ru 5012\n",
      "ab 4900\n",
      "ck 4831\n",
      "ki 4823\n",
      "ex 4745\n",
      "op 4713\n",
      "dr 4704\n",
      "ug 4669\n",
      "ga 4574\n",
      "ak 4570\n",
      "ok 4554\n",
      "gr 4535\n",
      "_j 4453\n",
      "da 4419\n",
      "ue 4411\n",
      "oi 4408\n",
      "ew 4397\n",
      "au 4336\n",
      "up 4301\n",
      "ef 4282\n",
      "ty 4261\n",
      "tl 4173\n",
      "ff 4154\n",
      "rm 4150\n",
      "lu 4132\n",
      "ci 4017\n",
      "ye 4006\n",
      "ds 3963\n",
      "rn 3742\n",
      "_q 3719\n",
      "by 3658\n",
      "gi 3641\n",
      "br 3627\n",
      "pp 3590\n",
      "cr 3543\n",
      "ui 3524\n",
      "uc 3404\n",
      "sc 3403\n",
      "cl 3347\n",
      "vo 3309\n",
      "va 3205\n",
      "nn 3200\n",
      "ny 3185\n",
      "wn 3176\n",
      "v_ 3058\n",
      "aw 3003\n",
      "kn 3001\n",
      "lf 2984\n",
      "eg 2768\n",
      "ft 2743\n",
      "mu 2734\n",
      "oc 2667\n",
      "fu 2642\n",
      "hr 2635\n",
      "ls 2621\n",
      "hu 2611\n",
      "ms 2517\n",
      "cu 2509\n",
      "nk 2496\n",
      "nl 2478\n",
      "sk 2471\n",
      "eo 2420\n",
      "lt 2410\n",
      "rc 2387\n",
      "gu 2374\n",
      "um 2364\n",
      "hy 2357\n",
      "ud 2344\n",
      "mm 2337\n",
      "pt 2303\n",
      "pu 2250\n",
      "af 2213\n",
      "sm 2141\n",
      "rg 2128\n",
      "du 2104\n",
      "fl 2038\n",
      "bi 2034\n",
      "gl 1999\n",
      "ik 1979\n",
      "rk 1949\n",
      "ys 1936\n",
      "eu 1929\n",
      "ua 1925\n",
      "sl 1884\n",
      "dy 1875\n",
      "ju 1870\n",
      "mb 1859\n",
      "oa 1839\n",
      "ps 1830\n",
      "tw 1819\n",
      "uk 1815\n",
      "ob 1807\n",
      "rl 1802\n",
      "je 1785\n",
      "ks 1729\n",
      "ib 1723\n",
      "ip 1676\n",
      "dd 1651\n",
      "xp 1610\n",
      "dl 1513\n",
      "iu 1510\n",
      "c_ 1495\n",
      "nu 1486\n",
      "gn 1485\n",
      "ya 1471\n",
      "gs 1441\n",
      "cc 1433\n",
      "sw 1410\n",
      "lk 1376\n",
      "nf 1361\n",
      "yi 1358\n",
      "ub 1352\n",
      "oy 1257\n",
      "ws 1226\n",
      "tc 1220\n",
      "rv 1203\n",
      "og 1104\n",
      "jo 1099\n",
      "ze 1081\n",
      "nv 1052\n",
      "xt 1028\n",
      "x_ 1026\n",
      "eh 997\n",
      "oe 987\n",
      "ae 922\n",
      "z_ 864\n",
      "yt 863\n",
      "ez 856\n",
      "wr 829\n",
      "iz 824\n",
      "ph 822\n",
      "rf 820\n",
      "kh 818\n",
      "ja 811\n",
      "rp 809\n",
      "xc 781\n",
      "ml 772\n",
      "dg 770\n",
      "gg 759\n",
      "bs 744\n",
      "xi 718\n",
      "à_ 714\n",
      "zo 701\n",
      "ku 695\n",
      "eb 695\n",
      "ka 680\n",
      "sn 659\n",
      "lm 658\n",
      "uz 652\n",
      "lv 640\n",
      "ko 632\n",
      "uf 632\n",
      "rb 628\n",
      "é_ 626\n",
      "oh 624\n",
      "sd 620\n",
      "xe 606\n",
      "lp 594\n",
      "sy 584\n",
      "_à 578\n",
      "lw 554\n",
      "dé 545\n",
      "_é 537\n",
      "vn 532\n",
      "lr 530\n",
      "lc 527\n",
      "ah 519\n",
      "sb 514\n",
      "kl 499\n",
      "az 497\n",
      "ém 481\n",
      "py 481\n",
      "bb 471\n",
      "cy 467\n",
      "dn 456\n",
      "ix 450\n",
      "hm 445\n",
      "xa 442\n",
      "ek 439\n",
      "_x 438\n",
      "wd 433\n",
      "b_ 425\n",
      "dm 419\n",
      "ii 418\n",
      "dv 406\n",
      "nq 405\n",
      "wl 397\n",
      "ux 397\n",
      "rw 395\n",
      "eq 391\n",
      "sf 377\n",
      "uo 368\n",
      "uv 362\n",
      "tm 361\n",
      "bt 359\n",
      "mn 350\n",
      "kp 317\n",
      "rh 315\n",
      "hs 315\n",
      "té 306\n",
      "nb 305\n",
      "sq 303\n",
      "zi 296\n",
      "cb 290\n",
      "ré 283\n",
      "dj 282\n",
      "tf 275\n",
      "ée 269\n",
      "tn 262\n",
      "vs 262\n",
      "yb 249\n",
      "ér 242\n",
      "_z 242\n",
      "gt 238\n",
      "êt 238\n",
      "vr 235\n",
      "ym 234\n",
      "oj 228\n",
      "ét 225\n",
      "yl 224\n",
      "tz 222\n",
      "hn 220\n",
      "yp 218\n",
      "vl 215\n",
      "aj 214\n",
      "bj 214\n",
      "nj 208\n",
      "ox 208\n",
      "nm 207\n",
      "ky 205\n",
      "sé 202\n",
      "nh 202\n",
      "yr 201\n",
      "vy 201\n",
      "gy 198\n",
      "hl 198\n",
      "fs 196\n",
      "j_ 190\n",
      "mf 189\n",
      "za 187\n",
      "èr 178\n",
      "ép 175\n",
      "yc 174\n",
      "mt 171\n",
      "oz 169\n",
      "xx 169\n",
      "né 168\n",
      "nw 161\n",
      "éc 161\n",
      "és 158\n",
      "zh 154\n",
      "np 152\n",
      "yw 149\n",
      "cq 146\n",
      "ln 144\n",
      "cs 136\n",
      "zu 132\n",
      "_ê 130\n",
      "én 130\n",
      "pc 127\n",
      "fy 127\n",
      "iq 125\n",
      "cd 124\n",
      "xv 121\n",
      "là 118\n",
      "nr 115\n",
      "lq 114\n",
      "ax 113\n",
      "vé 112\n",
      "yf 110\n",
      "mê 110\n",
      "lg 109\n",
      "êm 108\n",
      "kw 108\n",
      "lé 107\n",
      "df 106\n",
      "él 105\n",
      "pf 103\n",
      "zz 102\n",
      "lb 102\n",
      "ej 102\n",
      "kr 101\n",
      "gm 101\n",
      "dk 100\n",
      "wb 98\n",
      "hk 97\n",
      "ès 97\n",
      "hf 96\n",
      "mé 96\n",
      "nx 93\n",
      "pé 93\n",
      "gé 91\n",
      "yn 90\n",
      "aî 90\n",
      "mr 90\n",
      "rè 87\n",
      "ît 85\n",
      "xh 85\n",
      "dw 85\n",
      "iè 85\n",
      "uh 83\n",
      "tp 82\n",
      "rz 82\n",
      "sg 80\n",
      "sv 79\n",
      "vu 77\n",
      "hé 76\n",
      "bm 74\n",
      "hd 72\n",
      "hb 71\n",
      "hc 71\n",
      "wk 69\n",
      "ié 67\n",
      "gd 66\n",
      "ù_ 65\n",
      "où 65\n",
      "wf 64\n",
      "zm 63\n",
      "âm 62\n",
      "yu 61\n",
      "cé 61\n",
      "sr 61\n",
      "hh 59\n",
      "mc 59\n",
      "rq 59\n",
      "év 59\n",
      "zl 59\n",
      "dh 57\n",
      "_â 55\n",
      "dt 54\n",
      "ég 54\n",
      "bv 53\n",
      "dq 53\n",
      "oq 52\n",
      "ao 52\n",
      "uy 51\n",
      "éd 51\n",
      "wy 50\n",
      "pm 49\n",
      "ût 48\n",
      "rê 47\n",
      "kc 47\n",
      "uj 47\n",
      "aa 47\n",
      "hw 47\n",
      "pè 45\n",
      "ôt 45\n",
      "cè 43\n",
      "ué 42\n",
      "lh 41\n",
      "yk 40\n",
      "cx 40\n",
      "âc 39\n",
      "lè 38\n",
      "nz 37\n",
      "ço 37\n",
      "iw 36\n",
      "zy 35\n",
      "ww 35\n",
      "sû 34\n",
      "nê 34\n",
      "ûr 33\n",
      "dè 32\n",
      "km 32\n",
      "kb 32\n",
      "èn 32\n",
      "pw 32\n",
      "èl 31\n",
      "yd 30\n",
      "lx 30\n",
      "wu 30\n",
      "xy 29\n",
      "fw 29\n",
      "ji 28\n",
      "bd 28\n",
      "lz 27\n",
      "ât 27\n",
      "pk 27\n",
      "rj 27\n",
      "zd 27\n",
      "dc 27\n",
      "fb 27\n",
      "kg 27\n",
      "pç 26\n",
      "aq 25\n",
      "tb 25\n",
      "yh 25\n",
      "bh 25\n",
      "râ 25\n",
      "tô 25\n",
      "èv 23\n",
      "éa 23\n",
      "dp 23\n",
      "xl 22\n",
      "éf 22\n",
      "éj 21\n",
      "hè 21\n",
      "bé 21\n",
      "pg 20\n",
      "bw 20\n",
      "mè 20\n",
      "xu 20\n",
      "_ô 20\n",
      "wt 20\n",
      "kf 20\n",
      "oû 19\n",
      "tê 19\n",
      "gw 18\n",
      "rô 18\n",
      "db 18\n",
      "jà 18\n",
      "wc 18\n",
      "zn 17\n",
      "çu 17\n",
      "pê 17\n",
      "fâ 17\n",
      "xo 17\n",
      "tv 17\n",
      "eû 17\n",
      "tè 17\n",
      "èm 17\n",
      "îl 16\n",
      "ôl 16\n",
      "_î 16\n",
      "eç 16\n",
      "hâ 16\n",
      "ô_ 15\n",
      "fé 15\n",
      "pb 14\n",
      "èc 14\n",
      "êc 13\n",
      "èt 13\n",
      "yé 13\n",
      "tk 13\n",
      "yz 13\n",
      "bn 13\n",
      "gè 12\n",
      "în 12\n",
      "mw 12\n",
      "tg 12\n",
      "kv 12\n",
      "éb 11\n",
      "yg 10\n",
      "éq 10\n",
      "pn 10\n",
      "lâ 10\n",
      "aï 10\n",
      "td 10\n",
      "cô 10\n",
      "wm 9\n",
      "èd 9\n",
      "rç 9\n",
      "cv 9\n",
      "vè 9\n",
      "éi 9\n",
      "ih 9\n",
      "oë 8\n",
      "ët 8\n",
      "vk 8\n",
      "êl 7\n",
      "xq 7\n",
      "kt 7\n",
      "fk 7\n",
      "cw 7\n",
      "hp 7\n",
      "fû 7\n",
      "vê 6\n",
      "mk 6\n",
      "yv 6\n",
      "sè 6\n",
      "sj 6\n",
      "nç 6\n",
      "kd 6\n",
      "zè 5\n",
      "âg 5\n",
      "sz 5\n",
      "fn 5\n",
      "ça 5\n",
      "zv 5\n",
      "jr 5\n",
      "cz 5\n",
      "ïr 5\n",
      "ûm 4\n",
      "ij 4\n",
      "ên 4\n",
      "èg 4\n",
      "fî 4\n",
      "zk 4\n",
      "lû 4\n",
      "vô 4\n",
      "hv 4\n",
      "cm 3\n",
      "fh 3\n",
      "zs 3\n",
      "ûl 3\n",
      "pd 3\n",
      "tâ 3\n",
      "gf 3\n",
      "rû 3\n",
      "dû 3\n",
      "bc 3\n",
      "pû 3\n",
      "uï 3\n",
      "mv 3\n",
      "âl 3\n",
      "éu 3\n",
      "êv 3\n",
      "uu 3\n",
      "ân 3\n",
      "ï_ 3\n",
      "uw 3\n",
      "uê 3\n",
      "pâ 3\n",
      "èb 3\n",
      "ïs 3\n",
      "uq 3\n",
      "vg 3\n",
      "û_ 2\n",
      "éé 2\n",
      "vv 2\n",
      "dz 2\n",
      "wp 2\n",
      "aç 2\n",
      "bâ 2\n",
      "oé 2\n",
      "gê 2\n",
      "_è 2\n",
      "éo 2\n",
      "ôn 2\n",
      "q_ 2\n",
      "hê 2\n",
      "îm 2\n",
      "fm 2\n",
      "bî 2\n",
      "tx 2\n",
      "xé 2\n",
      "gb 2\n",
      "dâ 2\n",
      "ïe 2\n",
      "gâ 2\n",
      "dî 2\n",
      "oï 2\n",
      "îc 2\n",
      "oî 2\n",
      "vt 2\n",
      "gz 2\n",
      "èq 2\n",
      "fê 1\n",
      "ôd 1\n",
      "bf 1\n",
      "ûc 1\n",
      "rî 1\n",
      "kk 1\n",
      "vw 1\n",
      "nû 1\n",
      "ïl 1\n",
      "qv 1\n",
      "hg 1\n",
      "oê 1\n",
      "pô 1\n",
      "fp 1\n",
      "mg 1\n",
      "câ 1\n",
      "cg 1\n",
      "nâ 1\n",
      "wg 1\n",
      "âb 1\n",
      "iâ 1\n",
      "cp 1\n",
      "mô 1\n",
      "mq 1\n",
      "xm 1\n",
      "ïv 1\n",
      "iy 1\n",
      "cê 1\n",
      "éâ 1\n",
      "hq 1\n",
      "mâ 1\n",
      "zw 1\n",
      "fè 1\n",
      "zb 1\n",
      "cn 1\n",
      "xs 1\n",
      "ûn 1\n",
      "bû 1\n",
      "éh 1\n"
     ]
    }
   ],
   "source": [
    "for r,c in sorted(bigrams,key=lambda x:x[1],reverse=True):\n",
    "    print r,c\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16694"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigrams[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAHgCAYAAACW4wWWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu83HV95/H3O8lJTq5ATgIYTQjhJhIbJMFLJY8tqLvq\nwlaqVhddFrtd1qql1eI+1ltLechjsbL2smtb0UcfYsEutZaKta32AV4CaMkFAokh2BogQMGThNxv\n5/LZP2bYxyGcnDkz3znf72/OvJ6PRx6ZOTOf+Xxn5jfzme/v9nFECAAAlDOl9AAAAOh2FGMAAAqj\nGAMAUBjFGACAwijGAAAURjEGAKAwinGTbC+1van0OJ5n+zrb1xbIe43tLbZvy5y3La+/7ftKPUYb\nn8P+1MdAZ7J9ou0PlB4H2odijFZ9QNKbIuI9pQfSioj4+So8BiYH1+T8Pj1Rtc8gJomuLMa2/8b2\netubbV/dwkNMs31bfWb4V7ZnNZn/StsP2d5o+8+bTW77E7YftX2PpHNaiH+v7fttP2j7C7anNhn/\np5KWSfp72x9uIf+nbG+1fY/tv2hhZj/V9hfr7993bM9sYQzJs8o2PcYy2w/YvjD1scaRa6ntR2x/\nub783Gb7jbbvtf0T269u4nG2pLwHtj9ie1P932+2+DxSPoP//zPU4jL4/Di22v6KpE2SFjcRO9v2\nt+rfAZtsv6vJ9DdKOqP+Gf5sk7EvWjtj+1rb1zURf6PtD464Pu41dLY/avua+uXft313/fIl413T\nZvvC+ndob/213Gx7eRPjv37kcmf7Btu/0UT8++uv/YO2t9n+7nhjjysiuu6fpPn1/2eq9iHqayJ2\nqaSQ9Pr69T+TdG0T8edJelTSgpFjaSJ+paSHJc2SNE/SPzeZ/1xJ35TUU7/+x5KubOE1fOz559Bk\n3IWSHpTUK2mupJ80Of6lkgYlnV+//peS3tvCOPa3YTlq6THqz2GTaj+kHpC0Ikf+Ea/dK1X7Ib6+\nvvxa0i9K+psc78GIZXi2pDmSNkt6VZPPI+UzmPQZOmYcw5Je20Ls2yV9ccT1E1pZhlpZbkaLl3St\npOuaiH+VpO+PuP5jSYvHGftaSV+rX14j6X5JPZJ+R9J/a2IMn5Z0k6TPS/pYC89/Q/3yFEn/oibq\nwIjH6ak/h8tafS+e/9eVM2NJ19jeKOlHqv2aPavJ+O0RcW/98q2SLmoi9hLVFsQdkhQRu5rMvVrS\nHRFxMCL2Srqzyfg3qPZltNb2g/Xry5p8jBSvl/SNiDgcEftU+2HQrG0R8WD98nrVPlidZqGkb0h6\nT0RszJh3W0Q8HBHDqhXBu6L2rfKwmnsdU96Di1Rbhg9ExH5Jf63act2MlM9g6mdopMcj4kctxD0s\n6U22P2N7dUTsSRhDdhHxgKSTbS+yvULScxGxfZzh6yWttD1P0hFJP5S0SrX3ZU0Tw7he0pvqsb/X\nRJwi4jFJO22/StK/lfRAROxs5jHq/lDS3RHRyvfYC0xLfYBOY/sXJL1R0usi4qDt76k2S2vGsSf0\n7qQTfFvSLRHxsdIDSXBkxOUh1dZwdJo9kp5QrYj8OGPeka/d8Ijrw2ru+6D0e1CVz+CBVoIi4lHb\nF0h6q6RP274rIq5v79DGNKgXbqZs9jtQkr4m6R2STpV0+3iDImLA9jZJV0m6T9JDki6WdKakLU3k\n71NtzUqPauNv9r34Un0Mp6q2dqUptq+SdJqkDzUbO5punBmfoNqvuIO2X67aKpNmLbH9uvrlKyTd\n00Ts3ZLeabtPkmzPbzL3DyS9zfZM23MlXdZk/F2S3mH75Ofz2z6tycdIca+ky+rbeuZIujRj7io5\nKulySVfavqL0YDJbo9oyPMv2bNVeh2ZmRFLaZzD1M5TM9iJJByPiVkmflXRBkw+xT7XNPK16VrWZ\nbZ/tGWrtc3i7pHerVpC/1mTsGtVWjf+gfvn9qs1Om/lR9QVJn5J0m6TPNJlfku6Q9GbVNp19u5lA\n2ytVG/9762uZknXdzFjSP0h6v+0tkraqtqq6WVslfdD2n6k2q/mT8QZGxGbbN0j6vu0h1bYZXtVE\n/Abbt0vaKOlnktY2M/CI+LHtT0r6jmt7fw5I+qCkx5t5nFZFxFrbd6r2a/hZ1VbXddQquhGSZmMR\nccD2pZL+0fb+iEhZXdox6svwl1XbVihJX6qv9mxGymcw6TPUJq+U9Fnbw6p9Bn+tmeCI2Fnf8W6T\npL+PiI82GT9g+3rV3oOnJD3STHz9MTbXf8w8FRH/2mT4GkmfkPTD+ufgsJr4QWb7SkkDEfFV13ZA\nvc/2JRFxdxPjP1rf8Wp3RAw1Of4PSZov6bu2JWldRPxqk4/xAm7uhwiQzvaciNhf3wP2B5KujogN\npcfVjPqajQ0RkXOtAlTbE1jS30bEuPeebfB416m2M9xN7Xg8dIb6ZGSDpHdGxE9Kj6ejZsb1L8C7\nRrnpDS1ufEcZN9t+hWrbeW7pwEK8SNL3VNuTE0CHqX///K1qO/IVL8QSM2MAAIrrxh24AACoFIox\nAACFUYwBACisY4uxWzunNPFtiq/CGIjv7vgqjIH4zo6vyhikDi7GklJfAOLTlR4D8d0dX4UxEN/Z\n8VUZQ0cXYwAAJoXKHtq0YMGCWLp06XFv7+/v18KFC497++Dg4JiPv3PnTvX19R339mnTxj4Eu1H+\nRq/rjh07tGDBguPe/sQTT4wZf/jwYfX2Hv90snPnjn2mvL1792revHlj3qd+ZpmWH2NoaOyT2uzf\nv19z5sw57u0nnHDCmPGN3sOpU8fuDNnoPWj0/BstA/v27Rszfs+ePWM+x+nTp48Z/9xzz+mkk046\n7u2NlsFG8Y3ev927d+vEE08c8z49PT3HvW3Xrl2aP3/ss8E2eg8n+nO8f//YXTIbvQaHDh0aM77R\nZ2DGjBljxjdahmbOHPuU4Y3eg0afgUavf6NlaDzLwFjLcaNlWJKeeuqpMW8/cuTImK/zKaecMmZ8\no2Vg69atOyLi+AtZXWVP+rF06VKtW7eu5fj+/v6k/GMtYOMxPJx2utIPfSjt3OMXXdRME5vRNSoG\njTT6ImvkLW95S1J8ox8kjTT6Im9kzZpmT7f8QqedlnZyr0aFoJG9e/cmxUvSqaeemhTf6Iu6kdTP\n8T33NHPK6xfbtGlT4zuNYdmytIZqK1asSIpP/Qzs3r07KV5qPLFq5OMf/3hS/Ic/3HTL9hdYvXr1\nuE41zGpqAAAKoxgDAFAYxRgAgMIoxgAAFEYxBgCgMIoxAACFVaoY277a9jrb61IPTQIAoFNUqhhH\nxM0RsSoiVo11ID4AAJNJpYoxAADdiGIMAEBhFGMAAAqjGAMAUFi2RhG2+yTdNcpNb4iInbnGAQBA\n1WQrxvWCe36ufAAAdIrK9jP+uZ/7ufi7v/u7luNf9rKXJeV/+umnk+JT236ltp4bGBhIipfSW/CN\n1ct2PBr1IW3krLPOSop//PFxdT47rtQWiKnt53buTFvhdPrppyfFS+ltOJ988smk+MWLFyfF79mz\nJyk+tQVho37CjaS+/qnLYOr3qCQtWbIkKX727NlJ8UeOHEmKnzt37vqIWNXofmwzBgCgMIoxAACF\nUYwBACiMYgwAQGEUYwAACstWjG3/je31tjfbvjpXXgAAqi7bccaSfiUidtmeKWmt7a9zsg8AAPKu\npr7G9kZJP5K0WNKLDgId2c94165dGYcGAEA5WYqx7V+Q9EZJr4uIFZIekNR77P1G9jOeP39+jqEB\nAFBcrpnxCZKei4iDtl8u6bWZ8gIAUHm5ivE/SJpme4ukG1VbVQ0AAJRpB66IOCLpLTlyAQDQaTjO\nGACAwijGAAAURjEGAKCwyvYzvuCCC2LNmjUtx8+YMSMp/5Qpab9TUuORLnXZTu0lW1pqT+3UXrwS\n70FpQ0NDSfFTp05t00i6l236GQMA0AkoxgAAFEYxBgCgsFynw1xqe1OOXAAAdBpmxgAAFJazGE+z\nfZvtLbb/yvasjLkBAKisnMX4HEl/HBHnStor6QMZcwMAUFk5i/H2iLi3fvlWSRcde4eR/Yx37NiR\ncWgAAJSTsxgfe/T/i84GMLKf8YIFCzINCwCAsnIW4yW2X1e/fIWkezLmBgCgsnIW462SPljvaXyS\npD/JmBsAgMrK1c/4MUkvz5ELAIBOw3HGAAAURjEGAKAwijEAAIVl2WbcqpRepgMDA0m5Z86cmRRf\nuo9oai9bqT39bEs6ePBgUvzs2bPbNJIyUt+/AwcOJI8h9XNUup/x8PBwUnxqP+fS3wNV6Ge8ffv2\npPjFixe3aSQTi5kxAACFUYwBACiMYgwAQGEUYwAACqMYAwBQGMUYAIDCshZj21fafsj2Rtt/njM3\nAABVle1AUtvnSfqkpJ+PiB22549yn6slXS11zrFhAACkyjkzvkTS1yJihyRFxK5j70A/YwBAN2Kb\nMQAAheUsxndLeqftPkkabTU1AADdKNs244jYbPsGSd+3PSTpAUlX5coPAEBVZe0EEBG3SLolZ04A\nAKqObcYAABRGMQYAoLBKN6xN6WWa2k84VWof0NQ+qnv37k2Kl6T589nHrqTdu3cnxZ944olJ8T/9\n6U+T4iXp3HPPTYqfMqXsfOHo0aNJ8b29vW0aSWtKv36p/Zyl8uecaMdzGA9mxgAAFEYxBgCgMIox\nAACFUYwBACisSDG2fZ3ta0vkBgCgapgZAwBQWLZibPsTth+1fY+kc3LlBQCg6rIcZ2x7paR3Szq/\nnnODpPWj3I9+xgCArpNrZrxa0h0RcTAi9kq6c7Q70c8YANCN2GYMAEBhuYrxDyS9zfZM23MlXZYp\nLwAAlZdlm3FEbLB9u6SNkn4maW2OvAAAdIJsjSIi4gZJN+TKBwBAp2CbMQAAhVGMAQAojGIMAEBh\n2bYZN8u2pk6d2nL8zJkzk/J//etfT4pfunRpUnxqY/iXvexlSfGS1N/fnxR/4MCBpPg77xz1cPRx\ne9/73lc0f2pT8tNPPz0pfuPGjUnx5513XlK8JE2blvYVk/ocVqxYkRS/YcOGpPjTTjstKX769OlJ\n8bNnz06K/+hHP5oU/+u//utJ8ZI0Z86cpPjUWnDPPfckxY8XM2MAAAqjGAMAUBjFGACAwijGAAAU\nRjEGAKCwnP2M32v7ftsP2v6C7dZ3lQYAYBLJUoxtnyvpXZJeHxHnSxqS9J4cuQEAqLpcxxm/QdJK\nSWttS9JM1RpGvIDtqyVdLUlLlizJNDQAAMrKVYwt6ZaI+NhYd4qImyXdLEkrV65MO2MCAAAdItc2\n47skvcP2yZJke77ttFPTAAAwSWQpxhHxY0mflPQd2w9J+kdJL8mRGwCAqsvZz/h2SbfnygcAQKfg\nOGMAAAqjGAMAUBjFGACAwpzac3WiXHDBBXHvvfe2HN/b25uUv348dMcaHh5OfozUfsRz585NHkOK\ngYGBpPienp42jaQ1g4ODSfGpvYTb8d2wa9eupPi+vr7kMXSz1Pew078Hq8D2+ohY1eh+zIwBACiM\nYgwAQGEUYwAACstejG1fY3uL7dty5wYAoIqynfRjhA9IemNEPFkgNwAAlZN1Zmz7TyUtk/T3tj+c\nMzcAAFWVdWYcEe+3/WZJF0fEjpy5AQCoqkrtwGX7atvrbK/bsYNaDQDoDpUqxhFxc0SsiohVCxYs\nKD0cAACyqFQxBgCgG1GMAQAoLPuhTRGxNHdOAACqjJkxAACFUYwBACiMYgwAQGElToc5LraT+skO\nDQ0l5U/tBZuqCn1IU/sRp/ZUnjIl7bdiO3o6l1R6GXziiSeSH+O0005rw0jKKb0Mpzp06FBS/KxZ\ns9o0EjTCzBgAgMIoxgAAFEYxBgCgMIoxAACFUYwBACgs2+6atj8l6b2S+iVtl7Q+Im7KlR8AgKrK\nUoxtXyjp7ZJWSOqRtEHS+hy5AQCoulyrqV8v6RsRcTgi9kn65mh3op8xAKAbVWqbMf2MAQDdKFcx\nvlfSZbZ7bc+RdGmmvAAAVF6WbcYRsdb2nZIekvSspIcl7cmRGwCAqsu5mvqmiDhb0r+TdJrYgQsA\nAEl5G0XcbPsVknol3RIRGzLmBgCgsrIV44i4IlcuAAA6SaX2pgYAoBtVtp/x0NCQ9uxpfR+vvr6+\nNo6meZs2bUqKX758eZtGUs727duT4lN74c6YMSMpPtXRo0eT4qdPn96mkbSmCr2IU/uST506NSk+\ntR9xyneYJN19991J8ZdffnlSfKoNG9K3Rq5YsSIpPnUZOHjwYFL8eDEzBgCgMIoxAACFUYwBACgs\nSzG2vdR22kZUAAAmKWbGAAAUlrMYT7X9RdubbX/H9syMuQEAqKycxfgsSZ+PiPMk7VatvzEAAF0v\nZzHeFhEP1i+vl7T02DuM7Ge8c+fOjEMDAKCcnMX4yIjLQxrlhCMj+xmXPmkHAAC5sAMXAACFUYwB\nACgsy7mpI+IxSctHXL8pR14AADoBM2MAAAqjGAMAUBjFGACAwirbz9i2enp6Wo4/cOBAUv7UPqbL\nli1Lin/uueeS4ufNm5cUL0k7duxIik89PO3RRx9Nij/77LOT4iMiKf7xxx9Pik99D1P7uM6ZMycp\nXlLSZ1iS+vv7k+JPPfXUovlTe2pfcsklSfGHDx9Oiv/pT3+aFD9tWnqJeeaZZ5LiTznllKR4+hkD\nANAlKMYAABRGMQYAoLAixdj2fSXyAgBQRUWKcUT8fIm8AABUUamZ8f4SeQEAqCK2GQMAUFilijH9\njAEA3ahSxZh+xgCAblSpYgwAQDeiGAMAUFipQ5vST3oLAMAkwcwYAIDCKMYAABRGMQYAoDCn9myd\nKKtWrYq1a9e2HG87KX/q65KaHwDQ+Wyvj4hVje7HzBgAgMIoxgAAFEYxBgCgsGLFmJ7GAADUFCvG\n9DQGAKCm5MyYnsYAAIhtxgAAFFepYjyyn3F/f3/p4QAAkEWlivHIfsYLFy4sPRwAALKoVDEGAKAb\nlSzG1TwPJwAAmRUpxrb7JO0qkRsAgKrJXoxtL5L0Q0k35c4NAEAVTcudMCKelnR27rwAAFQVO3AB\nAFBY9plxLkNDQ0nxU6dOTYrft29fUvzcuXOT4tH5PamHh4eT4qdMSfutnZpfSn8PUj+HqdrxGqRI\nfQ8HBweT4qdNm7QlonKYGQMAUBjFGACAwijGAAAUlq0Y215qe1OufAAAdApmxgAAFFbqDFzLbD9g\n+8IS+QEAqJLs+63bPkfS/5V0VURszJ0fAICqyT0zXijpG5LeM1ohpp8xAKAb5S7GeyQ9Iemi0W6k\nnzEAoBvlXk19VNLlkr5te39EfDVzfgAAKqdEo4gDti+V9I/1gnxn7jEAAFAl2YpxRDwmaXn98m5J\n7EkNAIA4zhgAgOIoxgAAFEYxBgCgsMo2q4wIHTlypOX4Q4cOJeVPjT/ppJOS4lP7qKa8ds8r3Ut2\n+vTpSfH79+9Pik/tKZ3aU3vbtm1J8cuWLUuKb4ejR48mxaf2BT/55JOT4lOl9sROXYZ2796dFN/X\n15cU3w6l+4qn9oQeL2bGAAAURjEGAKAwijEAAIVRjAEAKKxUC8W0PWsAAJhEmBkDAFAYxRgAgMIq\nVYzpZwwA6EaVKsb0MwYAdKNKFWMAALoRxRgAgMIoxgAAFNawUYTtXkmXSlotaZGkQ5I2SfpWRGxu\nJWlEzGklDgCAyWjMYmz7d1UrxN+T9E+SfiapV9LZkm6sF+rfioiHJnicAABMWo1mxvdHxO8c57bP\n2T5Z0pI2jwkAgK4yZjGOiG81uP1nqs2W2y4iknr6zp49Oyl/aj/i0qZNS29V3dPT04aRlFO6D2pq\nP+gzzjgjKX5gYCApPrWfdDvGULofcWpf8SlT0nbLSV2GTjjhhKT40p+hbtLykmL75nYOBACAbtVo\nm/H8490k6a3tHw4AAN2n0brMfkmPq1Z8nxf162XXHwEAMEk0KsY/lfSGiHji2Btsb5+YIQEA0F0a\nbTP+A0nH25Pp99o8FgAAulKjvak/P8Zt/7v9wwEAoPs0vTf1ePeitr3U9iO2v2z7Udu32X6j7Xtt\n/8T2q5sfLgAAk08rhzatauK+Z0r6X5JeXv93haSLJF0r6ePH3nlkP+MdO3a0MDQAADpPK8W4mZN8\nbIuIhyNiWNJmSXdFREh6WNLSY+88sp/xggULWhgaAACdZ1zF2PY7n78cEW8+9m9jODLi8vCI68Ma\nR5MKAAC6wXhnxh8b598AAECTGp2B6y2qnWnrpbb/aMRN8yQNTuTAAADoFo1WFT8tab2k/1D//3n7\nJH14rMCIeEzS8hHXrzrebQAAdLNGxxlvlLTR9m0RkdZ+BQAAjGrMbca2v2n7suPctsz29bZ/ZWKG\nBgBAd2i0mvq/SvqIpD+wvUu1xhG9kk6X9M+S/k9EfGNihwgAwOTWaDX1M5L+u+1bJB2Q9BJJhyQ9\nKunVEfG9iRqYbU2b1vrRTw8++GBS/le/urNPELZ79+7kx1i4cGFSfGpj+Z6enqT4tWvXJsVffPHF\nSfHPPvtsUvwpp5ySFD99+vSk+KGhoaR4Serv70+KnzdvXvIYSpoypeWW8W3xzDPPJMUvWrQoKf7o\n0aNJ8ZI0c+bM5MdIMXXq1Cx5xruk3C7plyX9SNJWSZ+R9D8nalAAAHST8Rbj10haIuk+SWtV28v6\n9RM1KAAAusl4i/GAaqunZ6q2zXhb/RSXAAAg0XiL8VrVivGFklZL+o+2vzZhowIAoIuMdw+p/xIR\n6+qX/1XSL9r+TxM0JgAAusq4ZsYjCvHIv/35eJPUextvsf1F25ttf8d22V3kAACoiJz73Z8l6fMR\ncZ6k3ZLefuwd6GcMAOhGOYvxtoh4/uDf9aKfMQAAkvIW45G9jYdEP2MAACTlLcYAAGAUFGMAAArL\nsqp4lN7GN+XICwBAJ2BmDABAYRRjAAAKoxgDAFBYZQ8vGh4e1uHDh1uOT+1H/OSTTybFp/YRPfvs\ns5Pi23Gc9iOPPJIU39fXlzyGFKnLQGpP7OXLlze+0xhSl6HBwcGk+Hb0kU3th7t9+/ak+MWLFyfF\np/ZjnjNnTlL8vn37kuJTn3/qyZciIileSl8Ot2zZkhR/5MiRxndqA2bGAAAURjEGAKAwijEAAIVR\njAEAKIxiDABAYdmKse2P2N5U//ebufICAFB1WQ5tsr1S0vskvUaSJf2T7e9HxAM58gMAUGW5ZsYX\nSbojIg5ExH5Jfy1p9bF3sn217XW21+3cuTPT0AAAKKtS24wj4uaIWBURq0qfMAIAgFxyFeM1kt5m\ne5bt2ZIur/8NAICul6uF4gbbX5Z0f/1PX2J7MQAANdnOTR0Rn5P0uVz5AADoFJXaZgwAQDeiGAMA\nUBjFGACAwtyOfpMTYeXKlXHvvfe2HD9tWtrm8NT40oaHh5MfY8qUzv6tlrps206KT30PUl//1F64\nc+fOTYqX0nsqHzp0KCm+Hc8hxcDAQFJ8T09PUnzq69eOntap1qxJO/Bm9eoXndIiK9vrI2JVo/t1\n9rctAACTAMUYAIDCKMYAABRGMQYAoDCKMQAAhWUpxraX2n7E9m22t9j+K9uzcuQGAKDqcs6Mz5H0\nxxFxrqS9kj6QMTcAAJWVsxhvj4jnDxy+VbUexy8wsp9xf39/xqEBAFBOzmJ87BkYXnRGhpH9jBcu\nXJhpWAAAlJWzGC+x/br65Ssk3ZMxNwAAlZWzGG+V9EHbWySdJOlPMuYGAKCycp6AeTAi3psxHwAA\nHYHjjAEAKCzLzDgiHpO0PEcuAAA6DTNjAAAKq2zTXtuaMWNGUnw36/RexO1Qehko/R6U7uUrpfcF\nr8JzSJHajzhVFfoRpyrdjzgXvrEBACiMYgwAQGEUYwAACqMYAwBQGMUYAIDCshVj25+w/ajte2z/\nhe1rc+UGAKDKshzaZHulpHdLOr+ec4Ok9TlyAwBQdblmxqsl3RERByNir6Q7R7sT/YwBAN2oUtuM\n6WcMAOhGuYrxDyS9zfZM23MlXZYpLwAAlZerUcQG27dL2ijpZ5LW5sgLAEAnyLaaOiJuiIizI+Ii\nSY/mygsAQNVVapsxAADdqEjXpoi4rkReAACqiJkxAACFVbafcURoYGCg5fidO3cm5e/t7S0an9oL\n96mnnkqKl6RZs2Ylxaf2Uk3tZXvkyJGk+MnQC7a0p59+Oil+3759SfHnnHNOUnzqMpTSk70d+Z98\n8smk+GXLliXFR0RSvFS+L3gu3fEsAQCoMIoxAACFUYwBACiMYgwAQGFFi7Fr+EEAAOhq2Quh7aW2\nt9r+iqRNkhbnHgMAAFVS6tCmsyT954j4UaH8AABURqlVxI+PVohH9jPesWNHiXEBAJBdqWJ8YLQ/\njuxnvGDBgtxjAgCgCHaeAgCgMIoxAACFZd+BKyIek7Q8d14AAKqKmTEAAIVRjAEAKIxiDABAYZXt\nZzw0NKS9e/e2HJ/aA3PevHlJ8VOnTk2KP3r0aFL8SSedlBTfjscYHBxMik99D5999tmk+KVLlybF\np/biTe2nPDw8nBQ/NDSUFC9J8+fPT4ovfYhj6ucwNb7Te2q3o59xqtI9qceLmTEAAIVRjAEAKIxi\nDABAYRRjAAAKoxgDAFBYtmJse7btb9neaHuT7Xflyg0AQJXlPLTpzZKejoh/L0m2T8iYGwCAysq5\nmvphSW+y/RnbqyNiz7F3GNnPeOfOnRmHBgBAOdmKcUQ8KukC1Yryp23/9ij3+f/9jPv6+nINDQCA\norKtpra9SNKuiLjV9m5Jv5orNwAAVZZzm/ErJX3W9rCkAUm/ljE3AACVla0YR8S3JX07Vz4AADoF\nxxkDAFAYxRgAgMIoxgAAFFbpfsbPPfdcy/HTpqU9tdReunv2vOgw6qaccELaOVHa0Uf08OHDSfG9\nvb1J8al9SFPfw1Rz585Nik/tJzx9+vSk+GeeeSYpXpJmz56dFN/T05M8hhSp3yOp50so3dPadlJ8\nal93Sdq6dWtS/Jlnnpk8hhyYGQMAUBjFGACAwijGAAAUlrUY2z7R9gdy5gQAoOpyz4xPlEQxBgBg\nhNzF+EZJZ9h+0PZnM+cGAKCSch/a9D8kLY+I8zPnBQCgsiq1A9fIfsa7du0qPRwAALKoVDEe2c94\n/vz5pYdIIxlNAAAIMElEQVQDAEAWuYvxPklppyUCAGCSyVqMI2KnpHttb2IHLgAAarKfmzoirsid\nEwCAKqvUNmMAALoRxRgAgMIoxgAAFOZ29L2dCKtWrYr777+/5fjU5/XYY48lxS9evDgpPrUPaDt6\n+ab2E06V2o83tRfrHXfckRR/2WWXJcWnLsOp8amvvyRt27YtKX7RokVJ8an9gI8ePZoUn/oepH6O\nU/sxQzp06FBS/KxZs9ZHxKpG92NmDABAYRRjAAAKoxgDAFAYxRgAgMIoxgAAFJalGNteanvTiOvX\n2r4uR24AAKqOmTEAAIVVqhiP7Gfc399fejgAAGSRqxgPHpOrd7Q7jexnvHDhwjwjAwCgsFzF+FlJ\nJ9vusz1D0qWZ8gIAUHlZzpUWEQO2r5d0v6SnJD2SIy8AAJ0g24lLI+KPJP1RrnwAAHSKSu3ABQBA\nN6IYAwBQGMUYAIDCJm2zy6GhoaT4M844o00jKSP1+UtSb++oR6B1jIGBgaT4X/qlX2rTSFqT2gs3\ntZ9zO3qdv/SlL02KnzFjRlJ86Z7Qqe8BpMHBwaT41J7OqT2xx4uZMQAAhVGMAQAojGIMAEBhFGMA\nAAqjGAMAUBjFGACAwrIUY9s32v7giOvX2b42R24AAKou18z4dkm/POL6L9f/9gL0MwYAdKMsxTgi\nHlCtheIi2yskPRcR20e5H/2MAQBdJ+cZuL4m6R2STtUos2IAALpVzmJ8u6QvSlog6d9kzAsAQKVl\n25s6IjZLmivpqYj411x5AQCouqyNIiLilTnzAQDQCTjOGACAwijGAAAURjEGAKCwrNuMmzE0NKS9\ne/e2HD9v3rw2jqbz0NRc6unpKT2EJKXfw3bkHxgYSIqfMWNG8hhSpL4GEdGmkbSm9DLUDtOmVbZM\ntRUzYwAACqMYAwBQGMUYAIDCKMYAABRGMQYAoLAJLca2P2r7mvrl37d9d/3yJbZvm8jcAAB0iome\nGa+RtLp+eZWkObZ76n/7wbF3HtnPeMeOHRM8NAAAqmGii/F6SSttz5N0RNIPVSvKq1Ur1C8wsp/x\nggULJnhoAABUw4QeTR0RA7a3SbpK0n2SHpJ0saQzJW2ZyNwAAHSKHDtwrZF0rWqrpddIer+kB6L0\nqWkAAKiIXMX4JZJ+GBHPSjqsUVZRAwDQrSb8pJ8RcZeknhHXz57onAAAdBKOMwYAoDCKMQAAhVGM\nAQAorLKNIqdOnaoTTzyx9DAAJJgzZ07pIRQ1GfoJl3b06NGk+OnTp7dpJBOLmTEAAIVRjAEAKIxi\nDABAYRRjAAAKoxgDAFBYlmJs+0LbD9nutT3b9mbby3PkBgCg6rIc2hQRa23fKenTkmZKujUiNuXI\nDQBA1eU8zvh6SWtVaxRxzWh3sH21pKslacmSJflGBgBAQTm3GfdJmiNprqTe0e4QETdHxKqIWLVw\n4cKMQwMAoJycxfgLkj4l6TZJn8mYFwCASsuymtr2lZIGIuKrtqdKus/2JRFxd478AABUWa4duL4i\n6Sv1y0OSXpMjLwAAnYDjjAEAKIxiDABAYRRjAAAKq2w/Y0kaGhpqOXZwcDAp98GDB5PiU3sxR0RS\n/GQwPDycFD9lStnfmqnv4c6dO5PiZ8yYkRQ/a9aspHip/HuQ2k/48OHDSfG9vaMexTluKd+BUq0v\nfIrUZbgd/Zx7enqSHyNF6vfQeDEzBgCgMIoxAACFUYwBACiMYgwAQGEUYwAACsvVz/h627854voN\ntn8jR24AAKou18z4zyRdKUm2p0h6t6RbM+UGAKDScp2b+jHbO22/StIpkh6IiBcdREk/YwBAN8q5\nzfhLkq6S9D7VZsovQj9jAEA3ylmM75D0ZkkXSvp2xrwAAFRattNhRsRR29+VtLveRhEAAChjMa7v\nuPVaSe/MlRMAgE6Q69CmV0j6Z0l3RcRPcuQEAKBT5Nqb+seSluXIBQBAp+EMXAAAFFbpfsYpvThT\n+3im9oJN1Y4+oJ2udC/cVKnv4YIFC9o0ErQqtR9xqsnQjzhV6TEMDg5mydPZ33YAAEwCFGMAAAqj\nGAMAUBjFGACAwijGAAAUlq0Y236/7Qfr/7bVT40JAEDXy1aMI+JPI+J81RpFPCnpc7lyAwBQZSVW\nU/+hpLsj4pvH3mD7atvrbK/r7+8vMDQAAPLLWoxtXyXpNEm/O9rt9DMGAHSjnF2bVkq6VtLqiBjO\nlRcAgKrLOTP+kKT5kr5b34nrSxlzAwBQWdlmxhHxvly5AADoJBxnDABAYRRjAAAKoxgDAFCYU/td\nThTb/ZIeLz0OAAASnBYRDY/VrWwxBgCgW7CaGgCAwijGAAAURjEGAKAwijHQZWy/xPbf1i+fb/ut\nI2671Pb15UYHdCeKMdB9PiLpi/XL50t664jbviXpMtuzso8K6GIUY2CSsn2h7Yds99qebXuz7eWS\n3i7pH2xPl3S9pHfVzxf/rqgdXvE9SZcWHDrQdbKdmxpAXhGx1vadkj4taaakWyUdkPRcRByRJNu/\nLWlVRHxoROg6Sasl/WXmIQNdi2IMTG7XS1or6bCkayS9RlJ/g5ifSVo0weMCMALFGJjc+iTNkdQj\nqVfSofr/Y3n+fgAyYZsxMLl9QdKnJN0m6TOSHpW0dMTt+yTNPSbmbEmbcgwOQA3FGJikbF8paSAi\nvirpRkkXqraa+l9sn1m/23clveL5Hbjqf7tYtb2qAWTCuamBLmP7ckkrI+KTo9x2iqSvRsQb8o8M\n6F5sMwa6TETcYbvvODcvkfRbOccDgJkxAADFsc0YAIDCKMYAABRGMQYAoDCKMQAAhVGMAQAo7P8B\n+esPheFG8VUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112b8a650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "# Reduction table\n",
    "my2ascii_table = {\n",
    "    ord(u'â'):\"a\",\n",
    "    ord(u'ä'):\"e\",\n",
    "    ord(u\"à\"):\"a\",\n",
    "    ord(u\"æ\"):\"a\",\n",
    "    ord(u'ç'):\"c\",\n",
    "    ord(u\"é\"):\"e\",\n",
    "    ord(u\"è\"):\"e\",\n",
    "    ord(u\"ê\"):\"e\",\n",
    "    ord(u\"ë\"):\"e\",\n",
    "    ord(u'ğ'):\"g\",\n",
    "    ord(u'ı'):\"i\",\n",
    "    ord(u\"î\"):\"i\",\n",
    "    ord(u'ï'):\"i\",\n",
    "    ord(u'œ'):\"o\",\n",
    "    ord(u\"ô\"):\"o\",\n",
    "    ord(u'ö'):\"o\",\n",
    "    ord(u'ş'):\"s\",\n",
    "    ord(u'ù'):\"u\",\n",
    "    ord(u\"û\"):\"u\",\n",
    "    ord(u'ü'):\"u\",\n",
    "    ord(u'ß'):\"s\"\n",
    "    }\n",
    "\n",
    "\n",
    "def letter2idx(x):\n",
    "    if x=='_':\n",
    "        i = 0\n",
    "    else:\n",
    "        i = ord(x)-ord('a')+1\n",
    "    \n",
    "    if i<0 or i>26:\n",
    "        i = ord(my2ascii_table[ord(x)])-ord('a')+1\n",
    "        \n",
    "    return i\n",
    "\n",
    "T = np.zeros((27,27))\n",
    "# Convert bigrams to a transition matrix\n",
    "for pair in bigrams:\n",
    "    c = pair[1]\n",
    "    s = list(pair[0])\n",
    "    j = letter2idx(s[0])\n",
    "    i = letter2idx(s[1])\n",
    "    T[i,j] += c\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "\n",
    "alphabet=[chr(i+ord('a')) for i in range(26) ]\n",
    "alphabet.insert(0,'_')\n",
    "M = len(alphabet)\n",
    "\n",
    "plt.imshow(T/np.sum(T,axis=0), interpolation='nearest', vmin=0,cmap='gray_r')\n",
    "plt.xticks(range(M), alphabet)\n",
    "plt.xlabel('x(t)')\n",
    "plt.yticks(range(M), alphabet)\n",
    "plt.ylabel('x(t-1)')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.tick_top()\n",
    "#ax.set_title(f, va='bottom')\n",
    "plt.xlabel('x(t)')\n",
    "\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monte Carlo with Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pi is roughly 3.148096\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pyspark\n",
    "\n",
    "def sample(p):\n",
    "    x, y = 2*np.random.rand()-1, 2*np.random.rand()-1\n",
    "    return 1 if x*x + y*y < 1 else 0\n",
    "\n",
    "NUM_SAMPLES = 1000000\n",
    "\n",
    "count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample).reduce(lambda a, b: a + b)\n",
    "\n",
    "print(\"Pi is roughly %f\" % (4.0 * count / NUM_SAMPLES))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spark application: \n",
    "* a driver program that runs the user’s main function, that executes various parallel operations on a cluster. \n",
    "\n",
    "Main abstraction: \n",
    "* resilient distributed dataset (RDD), \n",
    "* a collection of elements partitioned across the nodes of the cluster that can be operated on in parallel. \n",
    "\n",
    "RDD: \n",
    "* Created from a file in the Hadoop file system (or any other Hadoop-supported file system)\n",
    "* Automatically recover from node failures.\n",
    "\n",
    "Cached: \n",
    "* Ask Spark to persist an RDD in memory\n",
    "\n",
    "Shared variables: \n",
    "* broadcast variables: cache a value in memory on all nodes, \n",
    "* accumulators: counters and sums.\n",
    "\n",
    "\n",
    "\n",
    "Spark’s Python API data formats (as of Nov 2016):\n",
    "\n",
    "* SparkContext.wholeTextFiles:\n",
    "read a directory containing of multiple small text files, return (filename, content) pairs. \n",
    "\n",
    "* RDD.saveAsPickleFile and SparkContext.pickleFile:\n",
    "save an RDD as pickled Python objects. \n",
    "\n",
    "* SequenceFile and Hadoop Input/Output Formats\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------------------+\n",
      "|   name|          properties|\n",
      "+-------+--------------------+\n",
      "|Product|[[Product identif...|\n",
      "+-------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "#    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "df = spark.read.json(\"data/products.json\")\n",
    "# Displays the content of the DataFrame to stdout\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|          properties|\n",
      "+--------------------+\n",
      "|[[Product identif...|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df.printSchema()\n",
    "df.select(\"properties\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading Parquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "AnalysisException",
     "evalue": "u'Path does not exist: file:/Users/cemgil/Downloads/sahibinden_NativeAdFeaturesEdr/dt=20171028/000000_0;'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAnalysisException\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-9de69b10c370>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/Users/cemgil/Downloads/sahibinden_NativeAdFeaturesEdr/dt=20171028/000000_0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/sql/readwriter.pyc\u001b[0m in \u001b[0;36mparquet\u001b[0;34m(self, *paths)\u001b[0m\n\u001b[1;32m    272\u001b[0m         \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'string'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'year'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'int'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'month'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'int'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'day'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'int'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    273\u001b[0m         \"\"\"\n\u001b[0;32m--> 274\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_seq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpaths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mignore_unicode_prefix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1133\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/sql/utils.pyc\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m     67\u001b[0m                                              e.java_exception.getStackTrace()))\n\u001b[1;32m     68\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'org.apache.spark.sql.AnalysisException: '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m': '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'org.apache.spark.sql.catalyst.analysis'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m': '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAnalysisException\u001b[0m: u'Path does not exist: file:/Users/cemgil/Downloads/sahibinden_NativeAdFeaturesEdr/dt=20171028/000000_0;'"
     ]
    }
   ],
   "source": [
    "df = spark.read.parquet(\"file\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[summary: string, flakeid: string, deliveryid: string, adid: string, version: string, channel: string, bidpricekurus: string, dailybudgetkurus: string, matchedadcity: string, adimpressioncount: string, adclickcount: string, adctr: string, adtitle: string, addescription: string, adcalltoaction: string, accountadcount: string, accountdayssincefirstadcreation: string, eventtype: string, vieweruniqueuserid: string, viewercity: string, vieweradimpressioncount: string, viewertotalimpressioncount: string, viewertotalclickcount: string, viewerip: string, trackingsinceseconds: string, vieweruserid: string, edrtypename: string, edrgroup: string, edrhostname: string, edruniqueid: string]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark (Spark 2.0.0)",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}