code/DreamWorks_code/Shrek_DataFrame.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NEW REPLACEMENT\n",
    "\n",
    "# Shrek Redone\n",
    "Shrek was my first attempt at parsing. It went okay, but from what I've learned of other scripts, I think I can do this better now. Here, I factor in white space from the start, remove parentheticals, and lower each entry. Finally, I add the relevant annotations found in my Disney data. For my very first attempt at the Shrek dataframe, go [here](https://github.com/Data-Science-for-Linguists-2019/Animated-Movie-Gendered-Dialogue/blob/master/code/DreamWorks_code/Shrek_Lines.ipynb). For my second attempt, go [here](https://github.com/Data-Science-for-Linguists-2019/Animated-Movie-Gendered-Dialogue/blob/master/code/DreamWorks_code/Shrek_Lines_DataFrame.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pretty printing has been turned OFF\n"
     ]
    }
   ],
   "source": [
    "%pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek = open(r'C:\\Users\\cassi\\Desktop\\Data_Science\\Animated-Movie-Gendered-Dialogue\\private\\imsdb_raw_nov_2015\\Animation\\shrek.txt')\n",
    "shrek_script = shrek.read()\n",
    "shrek.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'                                          SHREK\\n\\n                                       Written by\\n\\n                                William Steig & Ted Elliott\\n\\n                                     SHREK\\n\\n                         Once upon a time there was a lovely \\n\\n                         princes'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_script[:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\n                                     SHREK\\n\\n                         Once upon a time there was a lovely \\n\\n                         princes'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_script[159:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_script = shrek_script[159:] #this is where I want to start!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n\\n                                     SHREK\\n\\n                         Once upon a time there was a lovely \\n\\n                         princess. But she had an enchantment \\n\\n                         upon her of a fearful sort which could \\n\\n                         only be broken by love's first kiss.\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_script[:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take out all parentheticals (not lines)\n",
    "def no_parentheses(script):\n",
    "    new_script = re.sub(r\" *\\([^\\)]*\\)\", '', script)\n",
    "    return new_script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#my general function for analyzing any excessive white space\n",
    "def white_space_count(script_name):\n",
    "    white_space = re.findall(\" {3,}\", script_name)\n",
    "    len_w_s = [len(x) for x in white_space]\n",
    "    print(len_w_s[:100])\n",
    "    print(set(len_w_s))\n",
    "    for num in set(len_w_s):\n",
    "        print(num, \"white spaces appear\", len_w_s.count(num), \"times\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_script = no_parentheses(shrek_script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[37, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 15, 15, 15, 15, 37, 25, 37, 25, 37, 25, 25, 37, 25, 25, 15, 37, 25, 25, 25, 25, 37, 25, 37, 25, 25, 25, 37, 25, 15, 15, 15, 15, 37, 25, 25, 25, 15, 15, 15, 15, 15, 15, 15, 37, 25, 25, 37, 25, 37, 25, 37, 25, 25, 37, 25, 37, 25, 37, 37, 25, 25, 25, 37, 25, 37, 25, 37, 25, 37, 25, 37, 25, 37, 25, 25, 37, 25, 25, 15, 15, 37, 25, 37, 25, 37]\n",
      "{25, 37, 15}\n",
      "25 white spaces appear 1357 times\n",
      "37 white spaces appear 723 times\n",
      "15 white spaces appear 357 times\n"
     ]
    }
   ],
   "source": [
    "white_space_count(shrek_script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\n                                     SHREK\\n\\n                         Once upon a time there was a lovely \\n\\n                         princess. But she had an enchantment \\n\\n                         upon her of a fearful sort which could \\n\\n                         only be broken by love\\'s first kiss. \\n\\n                         She was locked away in a castle guarded \\n\\n                         by a terrible fire-breathing dragon. \\n\\n                         Many brave knights had attempted to \\n\\n                         free her from this dreadful prison, \\n\\n                         but non prevailed. She waited in the \\n\\n                         dragon\\'s keep in the highest room of \\n\\n                         the tallest tower for her true love \\n\\n                         and true love\\'s first kiss. \\n\\n                         Like that\\'s ever gonna happen. What \\n\\n                         a load of -\\n\\n               Allstar - by Smashmouth begins to play. Shrek goes about his \\n\\n               day. While in a nearby town, the villagers get together to go \\n\\n               after the ogre.\\n\\n               NIGHT - NEAR SHREK\\'S HOME\\n\\n                                     MAN1\\n\\n                         Think it\\'s in there?\\n\\n                                     MAN2\\n\\n                         All right. Let\\'s get it!\\n\\n                                     MAN1\\n\\n                         Whoa. Hold on. Do you know what that \\n\\n                         thing can do to you?\\n\\n                                     MAN3\\n\\n                         Yeah, it\\'ll grind your bones for it\\'s \\n\\n                         bread.\\n\\n               Shrek sneaks up behind them and laughs.\\n\\n                                     SHREK\\n\\n                         Yes, well, actually, that would be a \\n\\n                         giant. Now, ogres, oh they\\'re much worse. \\n\\n                         They\\'ll make a suit from your freshly \\n\\n                         peeled skin.\\n\\n                                     MEN\\n\\n                         No!\\n\\n                                     SHREK\\n\\n                         They\\'ll shave your liver. Squeeze the \\n\\n                         jelly from your eyes! Actually, it\\'s \\n\\n                         quite good on toast.\\n\\n                                     MAN1\\n\\n                         Back! Back, beast! Back! I warn ya! \\n\\n\\n\\n               Shrek calmly licks his fingers and extinguishes the torch. The \\n\\n               men shrink back away from him. Shrek roars very loudly and long \\n\\n               and his breath extinguishes all the remaining torches until the \\n\\n               men are in the dark.\\n\\n                                     SHREK\\n\\n                         This is the part where you run away. \\n\\n \\n\\n                         And stay out! \"Wanted. \\n\\n                         Fairy tale creatures.\"\\n\\n               THE NEXT DAY\\n\\n               There is a line of fairy tale creatures. The head of the guard \\n\\n               sits at a table paying people for'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_script[:3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 37 before new line\n",
    "# 25 between lines\n",
    "# 15 before scene descriptor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In my last attempt at Shrek code, I didn't consider the white space in front of the all-caps parts of text, making me overcapture things. This time, the white space is taken into account!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_list = re.findall(r\"\\n\\n {37}(\\b[A-Z]['A-Z& ]{2,}[0-9]?)+\", shrek_script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "722"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(char_list) #722 lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BIG BAD WOLF', 'BLIND MOUSE1', 'BLIND MOUSE2', 'CONGREGATION', 'CROWD', 'DONKEY', 'DWARF', 'FARQUAAD', 'FIONA', 'GINGERBREAD MAN', 'GIPETTO', 'GORDO', 'GUARD', 'GUARDS', 'HEAD GUARD', 'LITTLE BEAR', 'LITTLE PIG', 'MAN', 'MAN1', 'MAN2', 'MAN3', 'MEN', 'MERRY MAN', 'MERRY MEN', 'MIRROR', 'OLD WOMAN', 'PETER PAN', 'PINOCCHIO', 'PRIEST', 'ROBIN HOOD', 'SHREK', 'SHREK & FIONA', 'THELONIUS', 'WHISPERS', 'WOMAN', 'WOODEN PEOPLE']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(set(char_list)) #all characters who speak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Speakers are either followed by 2 or 4 newline characters; the \\n+ captures them all\n",
    "id_lines = re.sub(r\"\\n\\n {37}(\\b[A-Z]['A-Z& ]{2,}[0-9]?)+\\n+\", r\"_NEWLINE_\\1_\", shrek_script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"_NEWLINE_SHREK_                         Once upon a time there was a lovely \\n\\n                         princess. But she had an enchantment \\n\\n                         upon her of a fearful sort which could \\n\\n                         only be broken by love's first kiss. \\n\\n                         She was locked away in a castle guarded \\n\\n                         by a terrible fire-breathing dragon. \\n\\n                         Many brave knights had attempted to \\n\\n                         free her from this dreadful prison, \\n\\n                         but non prevailed. She waited in the \\n\\n                         dragon's keep in the highest room of \\n\\n                         the tallest tower for her true love \\n\\n                         and true love's first kiss. \\n\\n                         Like that's ever gonna happen. What \\n\\n                         a load of -\\n\\n               Allstar - by Smashmouth begins to play. Shrek goes about his \\n\\n               day. While in a nearby town, the villagers get together to go \\n\\n               after the ogre.\\n\\n               NIGHT - NEAR SHREK'S HOME_NEWLINE_MAN1_                         Think it's in there?_NEWLINE_MAN2_                         All right. Let's get it!_NEWLINE_MAN1_                         Whoa. Hold on. Do you know what that \\n\\n                         thing can do to you?_NEWLINE_MAN3_                         Yeah, it'll grind your bones for it's \\n\\n                         bread.\\n\\n               Shrek sneaks up behind them and laughs._NEWLINE_SHREK_                         Yes, well, actually, that would be a \\n\\n                         giant. Now, ogres, oh they're much worse. \\n\\n                         They'll make a suit from your freshly \\n\\n                         peeled skin._NEWLINE_MEN_                         No!_NEWLINE_SHREK_                         They'll shave your liver. Squeeze the \\n\\n                         jelly from your eyes! Actually, it's \\n\\n                         quite good on toast._NEW\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_lines[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get rid of gaps between the same set of lines\n",
    "id_lines_2 = re.sub(r\"\\n+ {25}\", '', id_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'_NEWLINE_SHREK_                         Once upon a time there was a lovely princess. But she had an enchantment upon her of a fearful sort which could only be broken by love\\'s first kiss. She was locked away in a castle guarded by a terrible fire-breathing dragon. Many brave knights had attempted to free her from this dreadful prison, but non prevailed. She waited in the dragon\\'s keep in the highest room of the tallest tower for her true love and true love\\'s first kiss. Like that\\'s ever gonna happen. What a load of -\\n\\n               Allstar - by Smashmouth begins to play. Shrek goes about his \\n\\n               day. While in a nearby town, the villagers get together to go \\n\\n               after the ogre.\\n\\n               NIGHT - NEAR SHREK\\'S HOME_NEWLINE_MAN1_                         Think it\\'s in there?_NEWLINE_MAN2_                         All right. Let\\'s get it!_NEWLINE_MAN1_                         Whoa. Hold on. Do you know what that thing can do to you?_NEWLINE_MAN3_                         Yeah, it\\'ll grind your bones for it\\'s bread.\\n\\n               Shrek sneaks up behind them and laughs._NEWLINE_SHREK_                         Yes, well, actually, that would be a giant. Now, ogres, oh they\\'re much worse. They\\'ll make a suit from your freshly peeled skin._NEWLINE_MEN_                         No!_NEWLINE_SHREK_                         They\\'ll shave your liver. Squeeze the jelly from your eyes! Actually, it\\'s quite good on toast._NEWLINE_MAN1_                         Back! Back, beast! Back! I warn ya! \\n\\n\\n\\n               Shrek calmly licks his fingers and extinguishes the torch. The \\n\\n               men shrink back away from him. Shrek roars very loudly and long \\n\\n               and his breath extinguishes all the remaining torches until the \\n\\n               men are in the dark._NEWLINE_SHREK_                         This is the part where you run away. \\n\\n And stay out! \"Wanted. Fairy tale creatures.\"\\n\\n               THE NEXT DAY\\n\\n               There is a line of fairy tale creatures. The head of the guard \\n\\n               sits at a table paying people for bringing the fairy tale creatures \\n\\n               to him. There are cages all around. Some of the people in line \\n\\n               are Peter Pan, who is carrying Tinkerbell in a cage, Gipetto \\n\\n               who\\'s carrying Pinocchio, and a farmer who is carrying the three \\n\\n               little pigs._NEWLINE_GUARD_                         All right. This one\\'s full. Take it away! Move it along. Come on! Get up!_NEWLINE_HEAD GUARD_                         Next!_NEWLINE_GUARD_ Give me that! Your flying days are over._NEWLINE_HEAD GUARD_                         That\\'s 20 pieces of silver for the witch. Next!_NEWLINE_GUARD_                         Get up! Come on!_NEWLINE_HEAD GUARD_                         Twenty pieces._NEWLINE_LITTLE BEAR_ This cage is too small._NEWLINE_DONKEY_                         Please, don\\'t turn me in. I\\'ll never be stubborn again. I can change. Please! Give me an'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_lines_2[:3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Fiona kiss...and the kiss fades into...\\n\\n               THE SWAMP\\n\\n               ...their wedding kiss. Shrek and Fiona are now married. 'I'm \\n\\n               a Believer' by Smashmouth is played in the background. Shrek \\n\\n               and Fiona break apart and run through the crowd to their awaiting \\n\\n               carriage. Which is made of a giant onion. Fiona tosses her bouquet \\n\\n               which both Cinderella and Snow White try to catch. But they end \\n\\n               up getting into a cat fight and so the dragon catches the bouquet \\n\\n               instead. The Gingerbread man has been mended somewhat and now \\n\\n               has one leg and walks with a candy cane cane. Shrek and Fiona \\n\\n               walk off as the rest of the guests party and Donkey takes over \\n\\n               singing the song._NEWLINE_GINGERBREAD MAN_                         God bless us, every one._NEWLINE_DONKEY_ Oh, that's funny. Oh. Oh. I can't breathe. I can't breathe.\\n\\n               THE END\\n\\n\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_lines_2[-1000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_lines = id_lines_2.split(\"_NEWLINE_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "723"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(shrek_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', \"SHREK_                         Once upon a time there was a lovely princess. But she had an enchantment upon her of a fearful sort which could only be broken by love's first kiss. She was locked away in a castle guarded by a terrible fire-breathing dragon. Many brave knights had attempted to free her from this dreadful prison, but non prevailed. She waited in the dragon's keep in the highest room of the tallest tower for her true love and true love's first kiss. Like that's ever gonna happen. What a load of -\\n\\n               Allstar - by Smashmouth begins to play. Shrek goes about his \\n\\n               day. While in a nearby town, the villagers get together to go \\n\\n               after the ogre.\\n\\n               NIGHT - NEAR SHREK'S HOME\", \"MAN1_                         Think it's in there?\", \"MAN2_                         All right. Let's get it!\", 'MAN1_                         Whoa. Hold on. Do you know what that thing can do to you?', \"MAN3_                         Yeah, it'll grind your bones for it's bread.\\n\\n               Shrek sneaks up behind them and laughs.\", \"SHREK_                         Yes, well, actually, that would be a giant. Now, ogres, oh they're much worse. They'll make a suit from your freshly peeled skin.\", 'MEN_                         No!', \"SHREK_                         They'll shave your liver. Squeeze the jelly from your eyes! Actually, it's quite good on toast.\", 'MAN1_                         Back! Back, beast! Back! I warn ya! \\n\\n\\n\\n               Shrek calmly licks his fingers and extinguishes the torch. The \\n\\n               men shrink back away from him. Shrek roars very loudly and long \\n\\n               and his breath extinguishes all the remaining torches until the \\n\\n               men are in the dark.']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_lines[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['SHREK_                         Really, really.', \"FIONA_ I love you too.\\n\\n               Shrek and Fiona kiss. Thelonius takes one of the cards and writes \\n\\n               'Awwww' on the back and then shows it to the congregation.\", \"CONGREGATION_                         Aawww!\\n\\n               Suddenly the magic of the spell pulls Fiona away. She's lifted \\n\\n               up into the air and she hovers there while the magic works around \\n\\n               her.\", 'WHISPERS_                         \"Until you find true love\\'s first kiss and then take love\\'s true form. Take love\\'s true form. Take love\\'s true form.\"\\n\\n               Suddenly Fiona\\'s eyes open wide. She\\'s consumed by the spell \\n\\n               and then is slowly lowered to the ground.', 'SHREK_ Fiona? Fiona. Are you all right?', \"FIONA_ Well, yes. But I don't understand. I'm supposed to be beautiful.\", 'SHREK_                         But you ARE beautiful.\\n\\n               They smile at each other.', \"DONKEY_ I was hoping this would be a happy ending.\\n\\n               Shrek and Fiona kiss...and the kiss fades into...\\n\\n               THE SWAMP\\n\\n               ...their wedding kiss. Shrek and Fiona are now married. 'I'm \\n\\n               a Believer' by Smashmouth is played in the background. Shrek \\n\\n               and Fiona break apart and run through the crowd to their awaiting \\n\\n               carriage. Which is made of a giant onion. Fiona tosses her bouquet \\n\\n               which both Cinderella and Snow White try to catch. But they end \\n\\n               up getting into a cat fight and so the dragon catches the bouquet \\n\\n               instead. The Gingerbread man has been mended somewhat and now \\n\\n               has one leg and walks with a candy cane cane. Shrek and Fiona \\n\\n               walk off as the rest of the guests party and Donkey takes over \\n\\n               singing the song.\", 'GINGERBREAD MAN_                         God bless us, every one.', \"DONKEY_ Oh, that's funny. Oh. Oh. I can't breathe. I can't breathe.\\n\\n               THE END\\n\\n\"]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_lines[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_lines = shrek_lines[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "722"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(shrek_lines) #matches up with number of lines from earlier!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#since all \\n characters are out of the actual dialogue, they only mark scene descriptions.\n",
    "#We can split each line of this and only take the first portion to get the actual lines of dialogue\n",
    "#Then it's just a getting rid of extra white space\n",
    "just_lines = []\n",
    "for line in shrek_lines:\n",
    "    just_line = line.split('\\n\\n')[0]\n",
    "    speaker_lines = just_line.split('_') #_ occurs after speaker name and before a line\n",
    "    speaker_lines[0] = speaker_lines[0].strip().lower()\n",
    "    speaker_lines[1] = speaker_lines[1].strip().lower()\n",
    "    just_lines.append(tuple(speaker_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "722"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(just_lines) #still 722!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('shrek', \"once upon a time there was a lovely princess. but she had an enchantment upon her of a fearful sort which could only be broken by love's first kiss. she was locked away in a castle guarded by a terrible fire-breathing dragon. many brave knights had attempted to free her from this dreadful prison, but non prevailed. she waited in the dragon's keep in the highest room of the tallest tower for her true love and true love's first kiss. like that's ever gonna happen. what a load of -\"), ('man1', \"think it's in there?\"), ('man2', \"all right. let's get it!\"), ('man1', 'whoa. hold on. do you know what that thing can do to you?'), ('man3', \"yeah, it'll grind your bones for it's bread.\"), ('shrek', \"yes, well, actually, that would be a giant. now, ogres, oh they're much worse. they'll make a suit from your freshly peeled skin.\"), ('men', 'no!'), ('shrek', \"they'll shave your liver. squeeze the jelly from your eyes! actually, it's quite good on toast.\"), ('man1', 'back! back, beast! back! i warn ya!'), ('shrek', 'this is the part where you run away.')]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "just_lines[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_df = pd.DataFrame(just_lines, columns=[\"Speaker\", \"Text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speaker</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>shrek</td>\n",
       "      <td>once upon a time there was a lovely princess. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>man1</td>\n",
       "      <td>think it's in there?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>man2</td>\n",
       "      <td>all right. let's get it!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>man1</td>\n",
       "      <td>whoa. hold on. do you know what that thing can...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>man3</td>\n",
       "      <td>yeah, it'll grind your bones for it's bread.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Speaker                                               Text\n",
       "0   shrek  once upon a time there was a lovely princess. ...\n",
       "1    man1                               think it's in there?\n",
       "2    man2                           all right. let's get it!\n",
       "3    man1  whoa. hold on. do you know what that thing can...\n",
       "4    man3       yeah, it'll grind your bones for it's bread."
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speaker</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>fiona</td>\n",
       "      <td>well, yes. but i don't understand. i'm suppose...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718</th>\n",
       "      <td>shrek</td>\n",
       "      <td>but you are beautiful.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>719</th>\n",
       "      <td>donkey</td>\n",
       "      <td>i was hoping this would be a happy ending.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>720</th>\n",
       "      <td>gingerbread man</td>\n",
       "      <td>god bless us, every one.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>donkey</td>\n",
       "      <td>oh, that's funny. oh. oh. i can't breathe. i c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Speaker                                               Text\n",
       "717            fiona  well, yes. but i don't understand. i'm suppose...\n",
       "718            shrek                             but you are beautiful.\n",
       "719           donkey         i was hoping this would be a happy ending.\n",
       "720  gingerbread man                           god bless us, every one.\n",
       "721           donkey  oh, that's funny. oh. oh. i can't breathe. i c..."
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Annotations\n",
    "While we're here, let's annotate with the necessary information (that found in the original Disney CSV file). This includes...\n",
    "\n",
    "1) Disney Period (which won't be included here because it isn't relevant to DreamWorks)\n",
    "\n",
    "2) Speaker Status (Princess, prince, or neither)\n",
    "\n",
    "3) Name of the movie\n",
    "\n",
    "4) Year the movie was made\n",
    "\n",
    "5) Utterance Number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['shrek', 'man1', 'man2', 'man3', 'men', 'guard', 'head guard',\n",
       "       'little bear', 'donkey', 'old woman', 'gipetto', 'pinocchio',\n",
       "       'peter pan', 'guards', 'blind mouse1', 'blind mouse2', 'gordo',\n",
       "       'dwarf', 'big bad wolf', 'little pig', 'farquaad',\n",
       "       'gingerbread man', 'mirror', 'thelonius', 'man', 'wooden people',\n",
       "       'crowd', 'woman', 'fiona', 'robin hood', 'merry men', 'merry man',\n",
       "       'shrek & fiona', 'priest', 'congregation', 'whispers'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_df.Speaker.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_princess = ['fiona']\n",
    "shrek_prince = ['farquaad']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "from functools import partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def which_stat(name, princess, prince):\n",
    "    if name in princess: return 'PRINCESS'\n",
    "    elif name in prince: return 'PRINCE'\n",
    "    else: return 'NON-P'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "stat_func = partial(which_stat, princess = shrek_princess, prince=shrek_prince)\n",
    "shrek_df[\"Speaker_Status\"] = shrek_df.Speaker.map(stat_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_df[\"Movie\"] = \"Shrek\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_df[\"Year\"] = 2001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_df[\"UTTERANCE_NUMBER\"] = shrek_df.Text.index + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speaker</th>\n",
       "      <th>Text</th>\n",
       "      <th>Speaker_Status</th>\n",
       "      <th>Movie</th>\n",
       "      <th>Year</th>\n",
       "      <th>UTTERANCE_NUMBER</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>shrek</td>\n",
       "      <td>once upon a time there was a lovely princess. ...</td>\n",
       "      <td>NON-P</td>\n",
       "      <td>Shrek</td>\n",
       "      <td>2001</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>man1</td>\n",
       "      <td>think it's in there?</td>\n",
       "      <td>NON-P</td>\n",
       "      <td>Shrek</td>\n",
       "      <td>2001</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>man2</td>\n",
       "      <td>all right. let's get it!</td>\n",
       "      <td>NON-P</td>\n",
       "      <td>Shrek</td>\n",
       "      <td>2001</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>man1</td>\n",
       "      <td>whoa. hold on. do you know what that thing can...</td>\n",
       "      <td>NON-P</td>\n",
       "      <td>Shrek</td>\n",
       "      <td>2001</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>man3</td>\n",
       "      <td>yeah, it'll grind your bones for it's bread.</td>\n",
       "      <td>NON-P</td>\n",
       "      <td>Shrek</td>\n",
       "      <td>2001</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Speaker                                               Text Speaker_Status  \\\n",
       "0   shrek  once upon a time there was a lovely princess. ...          NON-P   \n",
       "1    man1                               think it's in there?          NON-P   \n",
       "2    man2                           all right. let's get it!          NON-P   \n",
       "3    man1  whoa. hold on. do you know what that thing can...          NON-P   \n",
       "4    man3       yeah, it'll grind your bones for it's bread.          NON-P   \n",
       "\n",
       "   Movie  Year  UTTERANCE_NUMBER  \n",
       "0  Shrek  2001                 1  \n",
       "1  Shrek  2001                 2  \n",
       "2  Shrek  2001                 3  \n",
       "3  Shrek  2001                 4  \n",
       "4  Shrek  2001                 5  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shrek_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "shrek_df.to_pickle(r'..\\..\\..\\Animated-Movie-Gendered-Dialogue\\private\\shrek_lines.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}