Hobbit_SentimentAnalyses.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading collection 'popular'\n",
      "[nltk_data]    | \n",
      "[nltk_data]    | Downloading package cmudict to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package cmudict is already up-to-date!\n",
      "[nltk_data]    | Downloading package gazetteers to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package gazetteers is already up-to-date!\n",
      "[nltk_data]    | Downloading package genesis to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package genesis is already up-to-date!\n",
      "[nltk_data]    | Downloading package gutenberg to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package gutenberg is already up-to-date!\n",
      "[nltk_data]    | Downloading package inaugural to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package inaugural is already up-to-date!\n",
      "[nltk_data]    | Downloading package movie_reviews to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package movie_reviews is already up-to-date!\n",
      "[nltk_data]    | Downloading package names to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package names is already up-to-date!\n",
      "[nltk_data]    | Downloading package shakespeare to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package shakespeare is already up-to-date!\n",
      "[nltk_data]    | Downloading package stopwords to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package stopwords is already up-to-date!\n",
      "[nltk_data]    | Downloading package treebank to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package treebank is already up-to-date!\n",
      "[nltk_data]    | Downloading package twitter_samples to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package twitter_samples is already up-to-date!\n",
      "[nltk_data]    | Downloading package omw to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package omw is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package wordnet is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet_ic to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package wordnet_ic is already up-to-date!\n",
      "[nltk_data]    | Downloading package words to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package words is already up-to-date!\n",
      "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package maxent_ne_chunker is already up-to-date!\n",
      "[nltk_data]    | Downloading package punkt to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package punkt is already up-to-date!\n",
      "[nltk_data]    | Downloading package snowball_data to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package snowball_data is already up-to-date!\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package averaged_perceptron_tagger is already up-\n",
      "[nltk_data]    |       to-date!\n",
      "[nltk_data]    | \n",
      "[nltk_data]  Done downloading collection popular\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.text import Text\n",
    "from nltk.corpus import brown\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "nltk.download('popular')\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\">\n",
      "<!-- __ _ _ _ __| |_ (_)__ _____\n",
      "    / _` | '_/ _| ' \\| |\\ V / -_)\n",
      "    \\__,_|_| \\__|_||_|_| \\_/\\___| -->\n",
      "  <head>\n",
      "    <title>Full text of &quot;The Hobbit&quot;</title>\n",
      "\n",
      "          <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"/>\n",
      "    \n",
      "        <meta name=\"google-site-verification\" content=\"Q2YSouphkkgHkFNP7FgAkc4TmBs1Gmag3uGNndb53B8\" />\n",
      "    <meta name=\"google-site-verification\" content=\"bpjKvUvsX0lxfmjg19TLblckWkDpnptZEYsBntApxUk\" />\n",
      "\n",
      "    <script>\n",
      "/* @licstart  The following is the entire license notice for the\n",
      " * JavaScript code in this page.\n",
      " *\n",
      " * This program is free software: you can redistribute it and/or modify\n",
      " * it under the terms of the GNU Affero General Public License as published by\n",
      " * the Free Software Foundation, either version 3 of the License, or\n",
      " * (at your option) any later version.\n",
      " *\n",
      " * This program is distributed in the hope that it will be useful,\n",
      " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
      " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
      " * GNU Affero General Public License for more details.\n",
      " *\n",
      " * You should have received a copy of the GNU Affero General Public License\n",
      " * along with this program.  If not, see <http://www.gnu.org/licenses/>.\n",
      " *\n",
      " * @licend  The above is the entire license notice\n",
      " * for the JavaScript code in this page.\n",
      " */\n",
      "</script>\n",
      "        <script>window.archive_setup=[]</script>\n",
      "    <meta charset=\"UTF-8\">\n",
      "    <script src=\"//archive.org/includes/jquery-1.10.2.min.js?v1.10.2\" type=\"text/javascript\"></script>\n",
      "            <script src=\"//archive.org/includes/analytics.js?v=1cacd722\" type=\"text/javascript\"></script>\n",
      "    <script>\n",
      "      'use strict';\n",
      "      if ('archive_analytics' in window) {\n",
      "                  archive_analytics.service = \"ao_2\";\n",
      "        \n",
      "        archive_analytics.send_pageview_on_load({\"mediaType\":\"texts\"});\n",
      "\n",
      "        archive_analytics.process_url_events(window.location);\n",
      "      }\n",
      "    </script>\n",
      "        <script src=\"//archive.org/\n"
     ]
    }
   ],
   "source": [
    "r = requests.get('https://archive.org/stream/TheHobbitByJRRTolkienEBOOK/The%20Hobbit%20byJ%20%20RR%20Tolkien%20EBOOK_djvu.txt')\n",
    "\n",
    "# Setting the correct text encoding of the HTML page\n",
    "r.encoding = 'utf-8'\n",
    "\n",
    "# Extracting the HTML from the request object\n",
    "html = r.text\n",
    "\n",
    "# Printing the first 2000 characters in html\n",
    "print(html[:2000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chapter X IX \n",
      "\n",
      "\n",
      "THE LAST STAGE \n",
      "\n",
      "\n",
      "It was on May the First that the two came back \n",
      "at last to the brink of the valley of Rivendell, \n",
      "where stood the Last (or the First) Homely \n",
      "House. Again it was evening, their ponies were \n",
      "tired, especially the one that carried the \n",
      "baggage; and they all felt in need of rest. As \n",
      "they rode down the steep path, Bilbo heard the \n",
      "elves still singing in the trees, as if they had not \n",
      "stopped since he left; and as soon as the riders \n",
      "came down into the lower glades of the wood \n",
      "they burst into a song of much the same kind as \n",
      "before. This is something like it: \n",
      "\n",
      "\n",
      "The dragon is withered, \n",
      "\n",
      "His bones are nowcrumbled; \n",
      "His armour is shivered, \n",
      "\n",
      "His splendour is humbled! \n",
      "Though svnrd shall be rusted, \n",
      "And throne and crown perish \n",
      "With strength that men trusted \n",
      "And wealth that they cherish, \n",
      "Here grass is still growing, \n",
      "And leaves are yet swinging, \n",
      "The white water flowing, \n",
      "\n",
      "And elves are yet singing \n",
      "\n",
      "Come! Tra-la-la-lally! \n",
      "\n",
      "Come back to the valley! \n",
      "\n",
      "\n",
      "The stars are far brighter \n",
      "Than gems without measure, \n",
      "The moon is far whiter \n",
      "Than silver in treasure; \n",
      "\n",
      "The fire is more shining \n",
      "On hearth in the gloaming \n",
      "Than gold won by mining, \n",
      "\n",
      "So why go a-roaming? \n",
      "\n",
      "O! Tra-la-la-lally \n",
      "\n",
      "Come back to the Valley. \n",
      "\n",
      "\n",
      "O! Where are you going, \n",
      "\n",
      "So late in returning? \n",
      "\n",
      "The river is flowing, \n",
      "\n",
      "The stars are all burning! \n",
      "O! Whither so laden, \n",
      "\n",
      "So sad and so dreary? \n",
      "Here elf and elf-maiden \n",
      "Nowwelcome the weary \n",
      "With Tra-la-la-lally \n",
      "Come back to the Valley, \n",
      "Tra-la-la-lally \n",
      "Fa-1 a-la-l ally \n",
      "Fa-la! \n",
      "\n",
      "\n",
      "Then the elves of the valley came out and \n",
      "greeted them and led them across the water to \n",
      "the house of Elrond. There a warm welcome \n",
      "was made them, and there were many eager \n",
      "ears that evening to hear the tale of their \n",
      "adventures. Gandalf it was who spoke, for Bilbo \n",
      "was fallen quiet and drowsy. Most of the tale he \n",
      "knew, for he had been in it, and had himself told \n",
      "much of it to the wizard on their homeward way \n",
      "or in the house of Beorn; but every now and \n",
      "again he would open one eye, and listen, when \n",
      "a part of the story which he did not yet know \n",
      "came in. \n",
      "\n",
      "\n",
      "\n",
      "It was in this way that he learned where \n",
      "Gandalf had been to; for he overheard the \n",
      "words of the wizard to Elrond. It appeared that \n",
      "Gandalf had been to a great council of the white \n",
      "wizards, masters of lore and good magic; and \n",
      "that they had at last driven the Necromancer \n",
      "from his dark hold in the south of Mirkwood. \n",
      "\n",
      "“Ere long now,” Gandalf was saying, “the \n",
      "Forest will grow somewhat more wholesome. \n",
      "The North will be freed from that horror for many \n",
      "long years, I hope. Yet I wish he were banished \n",
      "from the world!” \n",
      "\n",
      "“It would be well indeed,” said Elrond; “but I \n",
      "fear that will not come about in this age of the \n",
      "world, or for many after.” \n",
      "\n",
      "When the tale of their journeyings was told, \n",
      "there were other tales, and yet more tales, tales \n",
      "of long ago, and tales of new things, and tales \n",
      "of no time at all, till Bilbo’s head fell forward on \n",
      "his chest, and he snored comfortably in a \n",
      "corner. \n",
      "\n",
      "He woke to find himself in a white bed, and \n",
      "the moon shining through an open window. \n",
      "Below it many elves were singing loud and \n",
      "clear on the banks of the stream. \n",
      "\n",
      "\n",
      "Sing all ye joyful, nowsing all together! \n",
      "The t fund’s in the tree-top, the wind’s in the \n",
      "heather; \n",
      "\n",
      "The stars are in blossom, the moon is in \n",
      "flower, \n",
      "\n",
      "And bright are the windows of Night in her \n",
      "tower. \n",
      "\n",
      "\n",
      "Dance all ye joyful, now dance all \n",
      "together! \n",
      "\n",
      "Soft is the grass, and let foot be like \n",
      "feather! \n",
      "\n",
      "The river is silver, the shadows are \n",
      "fleeting; \n",
      "\n",
      "\n",
      "\n",
      "Merry is May-time, and merry our \n",
      "meeting. \n",
      "\n",
      "\n",
      "Sing we now softly, and dreams let us \n",
      "leave him! \n",
      "\n",
      "Wind him in slumber and there let us \n",
      "leave him! \n",
      "\n",
      "The wanderer sleepeth. Now soft be his \n",
      "pillow/. \n",
      "\n",
      "Lullaby! Lullaby! Alder and Willow/ \n",
      "\n",
      "\n",
      "Sigh no more Pine, till the wind of the \n",
      "morn! \n",
      "\n",
      "Fall Moon! Dark be the land! \n",
      "\n",
      "Hush! Hush! Oak, Ash, and Thom! \n",
      "Hushed be all water, till dawn is at hand! \n",
      "\n",
      "\n",
      "“Well, Merry People!” said Bilbo looking \n",
      "out. “What time by the moon is this? Your lullaby \n",
      "would waken a drunken goblin! Yet I thank you.” \n",
      "\n",
      "“And your snores would waken a stone \n",
      "dragon — yet we thank you,” they answered with \n",
      "laughter. “It is drawing towards dawn, and you \n",
      "have slept now since the night’s beginning. \n",
      "Tomorrow, perhaps, you will be cured of \n",
      "weariness.” \n",
      "\n",
      "“A little sleep does a great cure in the \n",
      "house of Elrond,” said he; “but I will take all the \n",
      "cure I can get. A second good night, fair \n",
      "friends!” And with that he went back to bed and \n",
      "slept till late morning. \n",
      "\n",
      "Weariness fell from him soon in that house, \n",
      "and he had many a merry jest and dance, early \n",
      "and late, with the elves of the valley. Yet even \n",
      "that place could not long delay him now, and he \n",
      "thought always of his own home. After a week, \n",
      "\n",
      "\n",
      "\n",
      "therefore, he said farewell to Elrond, and giving \n",
      "him such small gifts as he would accept, he \n",
      "rode away with Gandalf. \n",
      "\n",
      "Even as they left the valley the sky darkened \n",
      "in the West before them, and wind and rain \n",
      "came up to meet them. \n",
      "\n",
      "“Merry is May-time!” said Bilbo, as the rain \n",
      "beat into his face. “But our back is to legends \n",
      "and we are coming home. I suppose this is the \n",
      "first taste of it.” \n",
      "\n",
      "“There is a long road yet,” said Gandalf. \n",
      "\n",
      "“But it is the last road,” said Bilbo. \n",
      "\n",
      "They came to the river that marked the very \n",
      "edge of the borderland of the Wild, and to the \n",
      "ford beneath the steep bank, which you may \n",
      "remember. The water was swollen both with the \n",
      "melting of the snows at the approach of \n",
      "summer, and with the daylong rain; but they \n",
      "crossed with some difficulty, and pressed \n",
      "forward, as evening fell, on the last stage of \n",
      "their journey. \n",
      "\n",
      "This was much as it had been before, \n",
      "except that the company was smaller, and more \n",
      "silent; also this time there were no trolls. At \n",
      "each point on the road Bilbo recalled the \n",
      "happenings and the words of a year ago — it \n",
      "seemed to him more like ten — so that, of \n",
      "course, he quickly noted the place where the \n",
      "pony had fallen in the river, and they had turned \n",
      "aside for their nasty adventure with Tom and \n",
      "Bert and Bill. \n",
      "\n",
      "Not far from the road they found the gold of \n",
      "the trolls, which they had buried, still hidden and \n",
      "untouched. “I have enough to last me my time,” \n",
      "said Bilbo, when they had dug it up. “You had \n",
      "better take this, Gandalf. I daresay you can find \n",
      "a use for it.” \n",
      "\n",
      "“Indeed I can!” said the wizard. “But share \n",
      "and share alike! You may find you have more \n",
      "needs than you expect.” \n",
      "\n",
      "So they put the gold in bags and slung them \n",
      "on the ponies, who were not at all pleased \n",
      "\n",
      "\n",
      "\n",
      "about it. After that their going was slower, for \n",
      "most of the time they walked. But the land was \n",
      "green and there was much grass through which \n",
      "the hobbit strolled along contentedly. He \n",
      "mopped his face with a red silk handkerchief — \n",
      "no! not a single one of his own had survived, he \n",
      "had borrowed this one from Elrond — for now \n",
      "June had brought summer, and the weather \n",
      "was bright and hot again. \n",
      "\n",
      "As all things come to an end, even this \n",
      "story, a day came at last when they were in \n",
      "sight of the country where Bilbo had been born \n",
      "and bred, where the shapes of the land and of \n",
      "the trees were as well known to him as his \n",
      "hands and toes. Coming to a rise he could see \n",
      "his own Hill in the distance, and he stopped \n",
      "suddenly and said: \n",
      "\n",
      "\n",
      "Roads go ever ever on, \n",
      "\n",
      "Over rock and under tree, \n",
      "\n",
      "By caves wh ere never sun has shone, \n",
      "\n",
      "By streams that never find the sea; \n",
      "Over snow by winter sown, \n",
      "\n",
      "And through the merry flowers of June, \n",
      "Overgrass and over stone, \n",
      "\n",
      "And under mountains in the moon. \n",
      "\n",
      "\n",
      "Roads go ever ever on \n",
      "\n",
      "Under cloud and under star, \n",
      "\n",
      "Yet feet that wandering have gone \n",
      "Turn at last to home afar. \n",
      "\n",
      "Eyes that fire and s/ord have seen \n",
      "And horror in the halls of stone \n",
      "Look at last on meadows green \n",
      "\n",
      "And trees and hills they long have \n",
      "known. \n",
      "\n",
      "\n",
      "\n",
      "Gandalf looked at him. “My dear Bilbo!” he \n",
      "said. “Something is the matter with you! You are \n",
      "not the hobbit that you were.” \n",
      "\n",
      "And so they crossed the bridge and passed \n",
      "the mill by the river and came right back to \n",
      "Bilbo’s own door. \n",
      "\n",
      "“Bless me! What’s going on?” he cried. \n",
      "There was a great commotion, and people of all \n",
      "sorts, respectable and unrespectable, were \n",
      "thick round the door, and many were going in \n",
      "and out — not even wiping their feet on the mat, \n",
      "as Bilbo noticed with annoyance. \n",
      "\n",
      "If he was surprised, they were more \n",
      "surprised still. He had arrived back in the \n",
      "middle of an auction! There was a large notice \n",
      "in black and red hung on the gate, stating that \n",
      "on June the Twenty-second Messrs Grubb, \n",
      "Grubb, and Burrowes would sell by auction the \n",
      "effects of the late Bilbo Baggins Esquire, of \n",
      "Bag-End, Underhill, Hobbiton. Sale to \n",
      "commence at ten o’clock sharp. It was now \n",
      "nearly lunchtime, and most of the things had \n",
      "already been sold, for various prices from next \n",
      "to nothing to old songs (as is not unusual at \n",
      "auctions). Bilbo’s cousins the Sackville- \n",
      "Bagginses were, in fact, busy measuring his \n",
      "rooms to see if their own furniture would fit. In \n",
      "short Bilbo was “Presumed Dead”, and not \n",
      "everybody that said so was sorry to find the \n",
      "presumption wrong. \n",
      "\n",
      "The return of Mr. Bilbo Baggins created \n",
      "quite a disturbance, both under the Hill and over \n",
      "the Hill, and across the Water; it was a great \n",
      "deal more than a nine days’ wonder. The legal \n",
      "bother, indeed, lasted for years. It was quite a \n",
      "long time before Mr. Baggins was in fact \n",
      "admitted to be alive again. The people who had \n",
      "got specially good bargains at the Sale took a \n",
      "deal of convincing; and in the end to save time \n",
      "Bilbo had to buy back quite a lot of his own \n",
      "furniture. Many of his silver spoons mysteriously \n",
      "disappeared and were never accounted for. \n",
      "\n",
      "\n",
      "\n",
      "Personally he suspected the Sackville- \n",
      "Bagginses. On their side they never admitted \n",
      "that the returned Baggins was genuine, and \n",
      "they were not on friendly terms with Bilbo ever \n",
      "after. They really had wanted to live in his nice \n",
      "hobbit-hole so very much. \n",
      "\n",
      "Indeed Bilbo found he had lost more than \n",
      "spoons — he had lost his reputation. It is true \n",
      "that for ever after he remained an elf-friend, and \n",
      "had the honour of dwarves, wizards, and all \n",
      "such folk as ever passed that way; but he was \n",
      "no longer quite respectable. He was in fact held \n",
      "by all the hobbits of the neighbourhood to be \n",
      "‘queer’ — except by his nephews and nieces on \n",
      "the Took side, but even they were not \n",
      "encouraged in their friendship by their elders. \n",
      "\n",
      "I am sorry to say he did not mind. He was \n",
      "quite content; and the sound of the kettle on his \n",
      "hearth was ever after more musical than it had \n",
      "been even in the quiet days before the \n",
      "Unexpected Party. His sword he hung over the \n",
      "mantelpiece. His coat of mail was arranged on \n",
      "a stand in the hall (until he lent it to a Museum). \n",
      "His gold and silver was largely spent in \n",
      "presents, both useful and extravagant — which to \n",
      "a certain extent accounts for the affection of his \n",
      "nephews and his nieces. His magic ring he \n",
      "kept a great secret, for he chiefly used it when \n",
      "unpleasant callers came. \n",
      "\n",
      "He took to writing poetry and visiting the \n",
      "elves; and though many shook their heads and \n",
      "touched their foreheads and said “Poor old \n",
      "Baggins!” and though few believed any of his \n",
      "tales, he remained very happy to the end of his \n",
      "days, and those were extraordinarily long. \n",
      "\n",
      "\n",
      "One autumn evening some years afterwards \n",
      "Bilbo was sitting in his study writing his \n",
      "memoirs — he thought of calling them “There \n",
      "and Back Again, a Hobbit’s Holiday” — when \n",
      "\n",
      "\n",
      "\n",
      "there was a ring at the door. It was Gandalf and \n",
      "a dwarf; and the dwarf was actually Balin. \n",
      "\n",
      "“Come in! Come in!” said Bilbo, and soon \n",
      "they were settled in chairs by the fire. If Balin \n",
      "noticed that Mr. Baggins’ waistcoat was more \n",
      "extensive (and had real gold buttons), Bilbo \n",
      "also noticed that Balin’s beard was several \n",
      "inches longer, and his jewelled belt was of great \n",
      "magnificence. \n",
      "\n",
      "They fell to talking of their times together, of \n",
      "course, and Bilbo asked how things were going \n",
      "in the lands of the Mountain. It seemed they \n",
      "were going very well. Bard had rebuilt the town \n",
      "in Dale and men had gathered to him from the \n",
      "Lake and from South and West, and all the \n",
      "valley had become tilled again and rich, and the \n",
      "desolation was now filled with birds and \n",
      "blossoms in spring and fruit and feasting in \n",
      "autumn. And Lake-town was refounded and \n",
      "was more prosperous than ever, and much \n",
      "wealth went up and down the Running River; \n",
      "and there was friendship in those parts \n",
      "between elves and dwarves and men. \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "The Hall at Bag-End Residence of B. \n",
      "Baggins Esquire \n",
      "\n",
      "\n",
      "The old Master had come to a bad end. \n",
      "Bard had given him much gold for the help of \n",
      "the Lake-people, but being of the kind that \n",
      "easily catches such disease he fell under the \n",
      "dragon-sickness, and took most of the gold and \n",
      "fled with it, and died of starvation in the Waste, \n",
      "\n",
      "\n",
      "\n",
      "deserted by his companions. \n",
      "\n",
      "“The new Master is of wiser kind,” said \n",
      "Balin, “and very popular, for, of course, he gets \n",
      "most of the credit for the present prosperity. \n",
      "They are making songs which say that in his \n",
      "day the rivers run with gold.” \n",
      "\n",
      "“Then the prophecies of the old songs have \n",
      "turned out to be true, after a fashion!” said \n",
      "Bilbo. \n",
      "\n",
      "“Of course!” said Gandalf. “And why should \n",
      "not they prove true? Surely you don’t disbelieve \n",
      "the prophecies, because you had a hand in \n",
      "bringing them about yourself? You don’t really \n",
      "suppose, do you, that all your adventures and \n",
      "escapes were managed by mere luck, just for \n",
      "your sole benefit? You are a very fine person, \n",
      "Mr. Baggins, and I am very fond of you; but you \n",
      "are only quite a little fellow in a wide world after \n",
      "all!” \n",
      "\n",
      "“Thank goodness!” said Bilbo laughing, and \n",
      "handed him the tobacco-jar.\n"
     ]
    }
   ],
   "source": [
    "# Creating a BeautifulSoup object from the HTML\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Getting the text out of the soup\n",
    "text = soup.get_text()\n",
    "\n",
    "# Fixing mistakes in text\n",
    "text = text.replace('v\\eave', 'leave')\n",
    "text = text.replace('mom!', 'morn!')\n",
    "\n",
    "# Entire Book\n",
    "book = text[9515:531263]\n",
    "\n",
    "# An Unexpected Party\n",
    "ch1 = book[0:47620] \n",
    "\n",
    "# Roast Mutton\n",
    "ch2 = book[47625:76488]\n",
    "\n",
    "# A Short Rest\n",
    "ch3 = book[76493:92164]\n",
    "\n",
    "# Over Hill and Under Hill\n",
    "ch4 = book[92169:114853]\n",
    "\n",
    "# Riddles in the Dark\n",
    "ch5 = book[114858:152917]\n",
    "\n",
    "# Out of the Frying-Pan into the Fire\n",
    "ch6 = book[152924:189738]\n",
    "\n",
    "# Queer Lodgings\n",
    "ch7 = book[189744:238305]\n",
    "\n",
    "# Flies And Spiders\n",
    "ch8 = book[238311:293647]\n",
    "\n",
    "# Barrels Out Of Bond\n",
    "ch9 = book[293653:325083]\n",
    "\n",
    "# A Warm Welcome\n",
    "ch10 = book[325089:346565]\n",
    "\n",
    "# On The Doorstep\n",
    "ch11 = book[346570:362945]\n",
    "\n",
    "# Inside Information \n",
    "ch12 = book[362950:401813]\n",
    "\n",
    "# Not At Home\n",
    "ch13 = book[401819:423373]\n",
    "\n",
    "# Fire And Water\n",
    "ch14 = book[423379:441167]\n",
    "\n",
    "# The Gathering Of The Clouds\n",
    "ch15 = book[441172:459450]\n",
    "\n",
    "# A Thief In The Night\n",
    "ch16 = book[459457:470910]\n",
    "\n",
    "# The Clouds Burst\n",
    "ch17 = book[470915:492687]\n",
    "\n",
    "# The Return Journey\n",
    "ch18 = book[492692:508103]\n",
    "\n",
    "# The Last Stage\n",
    "ch19 = book[508109:]\n",
    "\n",
    "print(ch19) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "508109"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find specific spots in the text\n",
    "book.find(\"Chapter X IX\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text\n",
       "0   Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...\n",
       "1   Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...\n",
       "2   Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...\n",
       "3   Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...\n",
       "4   Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...\n",
       "5   Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...\n",
       "6   Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...\n",
       "7   Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...\n",
       "8   Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...\n",
       "9   Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...\n",
       "10  Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...\n",
       "11  Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...\n",
       "12  Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...\n",
       "13  Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...\n",
       "14  Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...\n",
       "15  Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...\n",
       "16  Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...\n",
       "17  Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...\n",
       "18  Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "texts = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,ch18,ch19]\n",
    "\n",
    "data = pd.DataFrame(data=[text for text in texts], columns=['Text'])\n",
    "\n",
    "display(data.head(19))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>ROAST MUTTON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>A SHORT REST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...</td>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...</td>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...</td>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...</td>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...</td>\n",
       "      <td>A WARM WELCOME</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...</td>\n",
       "      <td>ON THE DOORSTEP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...</td>\n",
       "      <td>INSIDE INFORMATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...</td>\n",
       "      <td>NOT AT HOME</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...</td>\n",
       "      <td>FIRE AND WATER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...</td>\n",
       "      <td>THE GATHERING OF THE CLOUDS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...</td>\n",
       "      <td>A THIEF IN THE NIGHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...</td>\n",
       "      <td>THE CLOUDS BURST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...</td>\n",
       "      <td>THE RETURN JOURNEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...</td>\n",
       "      <td>THE LAST STAGE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text                             Chapters\n",
       "0   Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...                  AN UNEXPECTED PARTY\n",
       "1   Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...                         ROAST MUTTON\n",
       "2   Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...                         A SHORT REST\n",
       "3   Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...             OVER HILL AND UNDER HILL\n",
       "4   Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...                  RIDDLES IN THE DARK\n",
       "5   Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...  OUT OF THE FRYING-PAN INTO THE FIRE\n",
       "6   Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...                       QUEER LODGINGS\n",
       "7   Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...                    FLIES AND SPIDERS\n",
       "8   Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...                  BARRELS OUT OF BOND\n",
       "9   Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...                       A WARM WELCOME\n",
       "10  Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...                      ON THE DOORSTEP\n",
       "11  Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...                   INSIDE INFORMATION\n",
       "12  Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...                          NOT AT HOME\n",
       "13  Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...                       FIRE AND WATER\n",
       "14  Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...          THE GATHERING OF THE CLOUDS\n",
       "15  Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...                 A THIEF IN THE NIGHT\n",
       "16  Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...                     THE CLOUDS BURST\n",
       "17  Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...                   THE RETURN JOURNEY\n",
       "18  Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...                       THE LAST STAGE"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "chapters = ['AN UNEXPECTED PARTY','ROAST MUTTON','A SHORT REST','OVER HILL AND UNDER HILL','RIDDLES IN THE DARK',\n",
    "            'OUT OF THE FRYING-PAN INTO THE FIRE','QUEER LODGINGS','FLIES AND SPIDERS','BARRELS OUT OF BOND',\n",
    "            'A WARM WELCOME','ON THE DOORSTEP','INSIDE INFORMATION','NOT AT HOME','FIRE AND WATER',\n",
    "            'THE GATHERING OF THE CLOUDS','A THIEF IN THE NIGHT','THE CLOUDS BURST','THE RETURN JOURNEY',\n",
    "            'THE LAST STAGE']\n",
    "\n",
    "data1 = pd.DataFrame(chapters, columns=['Chapters'])\n",
    "\n",
    "data2 = pd.merge(data, data1, left_index=True, right_index=True)\n",
    "\n",
    "display(data2.head(28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.125155</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.004399</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.136103</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.034112</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.043253</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.024999</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.062156</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.036351</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.019908</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.095816</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.004927</td>\n",
       "      <td>0.060587</td>\n",
       "      <td>0.888353</td>\n",
       "      <td>0.051073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.068378</td>\n",
       "      <td>0.082650</td>\n",
       "      <td>0.816149</td>\n",
       "      <td>0.101219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.006529</td>\n",
       "      <td>0.057590</td>\n",
       "      <td>0.891776</td>\n",
       "      <td>0.050610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.016736</td>\n",
       "      <td>0.071964</td>\n",
       "      <td>0.853494</td>\n",
       "      <td>0.074536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.024960</td>\n",
       "      <td>0.081815</td>\n",
       "      <td>0.840382</td>\n",
       "      <td>0.077790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.070048</td>\n",
       "      <td>0.063877</td>\n",
       "      <td>0.856475</td>\n",
       "      <td>0.079656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.067713</td>\n",
       "      <td>0.096757</td>\n",
       "      <td>0.842087</td>\n",
       "      <td>0.061148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.083252</td>\n",
       "      <td>0.065162</td>\n",
       "      <td>0.832234</td>\n",
       "      <td>0.102604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.164969</td>\n",
       "      <td>0.044680</td>\n",
       "      <td>0.857906</td>\n",
       "      <td>0.097422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    compound       neg       neu       pos\n",
       "0   0.125155  0.046048  0.858940  0.095017\n",
       "1   0.004399  0.063258  0.874296  0.062453\n",
       "2   0.136103  0.035789  0.849283  0.114947\n",
       "3  -0.034112  0.082478  0.848184  0.069357\n",
       "4  -0.043253  0.103889  0.824226  0.071892\n",
       "5  -0.024999  0.079857  0.850042  0.070087\n",
       "6   0.062156  0.055134  0.858517  0.077430\n",
       "7  -0.036351  0.087934  0.843843  0.068238\n",
       "8   0.019908  0.075414  0.853004  0.071605\n",
       "9   0.095816  0.057529  0.862180  0.080308\n",
       "10 -0.004927  0.060587  0.888353  0.051073\n",
       "11  0.068378  0.082650  0.816149  0.101219\n",
       "12 -0.006529  0.057590  0.891776  0.050610\n",
       "13  0.016736  0.071964  0.853494  0.074536\n",
       "14  0.024960  0.081815  0.840382  0.077790\n",
       "15  0.070048  0.063877  0.856475  0.079656\n",
       "16 -0.067713  0.096757  0.842087  0.061148\n",
       "17  0.083252  0.065162  0.832234  0.102604\n",
       "18  0.164969  0.044680  0.857906  0.097422"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from nltk import tokenize\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer \n",
    "\n",
    "chapters = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,\n",
    "            ch18,ch19]\n",
    "\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "sentiments_list = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    sentence_list = tokenize.sent_tokenize(chapter)\n",
    "    sentiments = {'compound': 0.0, 'neg': 0.0, 'neu': 0.0, 'pos': 0.0}\n",
    "        \n",
    "    for sentence in sentence_list:\n",
    "        vs = analyzer.polarity_scores(sentence)\n",
    "        sentiments['compound'] += vs['compound']\n",
    "        sentiments['neg'] += vs['neg']\n",
    "        sentiments['neu'] += vs['neu']\n",
    "        sentiments['pos'] += vs['pos']\n",
    "            \n",
    "    sentiments['compound'] = sentiments['compound'] / len(sentence_list)\n",
    "    sentiments['neg'] = sentiments['neg'] / len(sentence_list)\n",
    "    sentiments['neu'] = sentiments['neu'] / len(sentence_list)\n",
    "    sentiments['pos'] = sentiments['pos'] / len(sentence_list)\n",
    "    \n",
    "    sentiments_list.append(sentiments)  # add this line\n",
    "\n",
    "data3 = pd.DataFrame(sentiments_list)  # add this line\n",
    "data3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...</td>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "      <td>-0.024999</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...</td>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "      <td>0.062156</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...</td>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "      <td>-0.036351</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...</td>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...</td>\n",
       "      <td>A WARM WELCOME</td>\n",
       "      <td>0.095816</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...</td>\n",
       "      <td>ON THE DOORSTEP</td>\n",
       "      <td>-0.004927</td>\n",
       "      <td>0.060587</td>\n",
       "      <td>0.888353</td>\n",
       "      <td>0.051073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...</td>\n",
       "      <td>INSIDE INFORMATION</td>\n",
       "      <td>0.068378</td>\n",
       "      <td>0.082650</td>\n",
       "      <td>0.816149</td>\n",
       "      <td>0.101219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...</td>\n",
       "      <td>NOT AT HOME</td>\n",
       "      <td>-0.006529</td>\n",
       "      <td>0.057590</td>\n",
       "      <td>0.891776</td>\n",
       "      <td>0.050610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...</td>\n",
       "      <td>FIRE AND WATER</td>\n",
       "      <td>0.016736</td>\n",
       "      <td>0.071964</td>\n",
       "      <td>0.853494</td>\n",
       "      <td>0.074536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...</td>\n",
       "      <td>THE GATHERING OF THE CLOUDS</td>\n",
       "      <td>0.024960</td>\n",
       "      <td>0.081815</td>\n",
       "      <td>0.840382</td>\n",
       "      <td>0.077790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...</td>\n",
       "      <td>A THIEF IN THE NIGHT</td>\n",
       "      <td>0.070048</td>\n",
       "      <td>0.063877</td>\n",
       "      <td>0.856475</td>\n",
       "      <td>0.079656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...</td>\n",
       "      <td>THE CLOUDS BURST</td>\n",
       "      <td>-0.067713</td>\n",
       "      <td>0.096757</td>\n",
       "      <td>0.842087</td>\n",
       "      <td>0.061148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...</td>\n",
       "      <td>THE RETURN JOURNEY</td>\n",
       "      <td>0.083252</td>\n",
       "      <td>0.065162</td>\n",
       "      <td>0.832234</td>\n",
       "      <td>0.102604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...</td>\n",
       "      <td>THE LAST STAGE</td>\n",
       "      <td>0.164969</td>\n",
       "      <td>0.044680</td>\n",
       "      <td>0.857906</td>\n",
       "      <td>0.097422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text                             Chapters  Compound  Negative   Neutral  Positive\n",
       "0   Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...                  AN UNEXPECTED PARTY  0.125155  0.046048  0.858940  0.095017\n",
       "1   Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...                         ROAST MUTTON  0.004399  0.063258  0.874296  0.062453\n",
       "2   Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...                         A SHORT REST  0.136103  0.035789  0.849283  0.114947\n",
       "3   Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...             OVER HILL AND UNDER HILL -0.034112  0.082478  0.848184  0.069357\n",
       "4   Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...                  RIDDLES IN THE DARK -0.043253  0.103889  0.824226  0.071892\n",
       "5   Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...  OUT OF THE FRYING-PAN INTO THE FIRE -0.024999  0.079857  0.850042  0.070087\n",
       "6   Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...                       QUEER LODGINGS  0.062156  0.055134  0.858517  0.077430\n",
       "7   Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...                    FLIES AND SPIDERS -0.036351  0.087934  0.843843  0.068238\n",
       "8   Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...                  BARRELS OUT OF BOND  0.019908  0.075414  0.853004  0.071605\n",
       "9   Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...                       A WARM WELCOME  0.095816  0.057529  0.862180  0.080308\n",
       "10  Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...                      ON THE DOORSTEP -0.004927  0.060587  0.888353  0.051073\n",
       "11  Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...                   INSIDE INFORMATION  0.068378  0.082650  0.816149  0.101219\n",
       "12  Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...                          NOT AT HOME -0.006529  0.057590  0.891776  0.050610\n",
       "13  Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...                       FIRE AND WATER  0.016736  0.071964  0.853494  0.074536\n",
       "14  Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...          THE GATHERING OF THE CLOUDS  0.024960  0.081815  0.840382  0.077790\n",
       "15  Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...                 A THIEF IN THE NIGHT  0.070048  0.063877  0.856475  0.079656\n",
       "16  Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...                     THE CLOUDS BURST -0.067713  0.096757  0.842087  0.061148\n",
       "17  Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...                   THE RETURN JOURNEY  0.083252  0.065162  0.832234  0.102604\n",
       "18  Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...                       THE LAST STAGE  0.164969  0.044680  0.857906  0.097422"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = pd.merge(data2, data3, left_index=True, right_index=True)\n",
    "\n",
    "Hobbit_sentiments=Hobbit_sentiments.rename(columns={\"compound\": \"Compound\", \"neg\": \"Negative\", \"neu\": \"Neutral\", \"pos\": \"Positive\"})\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "\n",
    "blob_book = TextBlob(book)\n",
    "blob_ch1 = TextBlob(ch1)\n",
    "blob_ch2 = TextBlob(ch2)\n",
    "blob_ch3 = TextBlob(ch3)\n",
    "blob_ch4 = TextBlob(ch4)\n",
    "blob_ch5 = TextBlob(ch5)\n",
    "blob_ch6 = TextBlob(ch6)\n",
    "blob_ch7 = TextBlob(ch7)\n",
    "blob_ch8 = TextBlob(ch8)\n",
    "blob_ch9 = TextBlob(ch9)\n",
    "blob_ch10 = TextBlob(ch10)\n",
    "blob_ch11 = TextBlob(ch11)\n",
    "blob_ch12 = TextBlob(ch12)\n",
    "blob_ch13 = TextBlob(ch13)\n",
    "blob_ch14 = TextBlob(ch14)\n",
    "blob_ch15 = TextBlob(ch15)\n",
    "blob_ch16 = TextBlob(ch16)\n",
    "blob_ch17 = TextBlob(ch17)\n",
    "blob_ch18 = TextBlob(ch18)\n",
    "blob_ch19 = TextBlob(ch19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.388771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.086392</td>\n",
       "      <td>0.337145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.019579</td>\n",
       "      <td>0.408200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.008257</td>\n",
       "      <td>0.363186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.090141</td>\n",
       "      <td>0.430428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.001852</td>\n",
       "      <td>0.360175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.066962</td>\n",
       "      <td>0.379169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.045913</td>\n",
       "      <td>0.357767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.060123</td>\n",
       "      <td>0.349235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.036574</td>\n",
       "      <td>0.364356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.041592</td>\n",
       "      <td>0.368692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.011715</td>\n",
       "      <td>0.288025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.068537</td>\n",
       "      <td>0.333555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.091542</td>\n",
       "      <td>0.298514</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Polarity  Subjectivity\n",
       "0   0.061535      0.391206\n",
       "1   0.047677      0.319128\n",
       "2   0.098664      0.358846\n",
       "3   0.004561      0.389319\n",
       "4  -0.006902      0.339751\n",
       "5   0.020121      0.388771\n",
       "6   0.086392      0.337145\n",
       "7   0.019579      0.408200\n",
       "8   0.008257      0.363186\n",
       "9   0.090141      0.430428\n",
       "10 -0.001852      0.360175\n",
       "11  0.066962      0.379169\n",
       "12  0.045913      0.357767\n",
       "13  0.060123      0.349235\n",
       "14  0.036574      0.364356\n",
       "15  0.041592      0.368692\n",
       "16  0.011715      0.288025\n",
       "17  0.068537      0.333555\n",
       "18  0.091542      0.298514"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blobs = [blob_ch1,blob_ch2,blob_ch3,blob_ch4,blob_ch5,blob_ch6,blob_ch7,\n",
    "         blob_ch8,blob_ch9,blob_ch10,blob_ch11,blob_ch12,blob_ch13,blob_ch14,blob_ch15,blob_ch16,blob_ch17,\n",
    "         blob_ch18,blob_ch19]\n",
    "\n",
    "polarity_list = list()\n",
    "subjectivity_list = list()\n",
    "\n",
    "for blob in blobs:\n",
    "    sentence_list1 = blob.sentences\n",
    "    polarity = (0)\n",
    "    subjectivity = (0)\n",
    "        \n",
    "    for sentence in sentence_list1:\n",
    "        pl=sentence.sentiment.polarity\n",
    "        polarity += pl\n",
    "        sb=sentence.sentiment.subjectivity\n",
    "        subjectivity += sb\n",
    "            \n",
    "    polarity = polarity / len(sentence_list1)\n",
    "    subjectivity = subjectivity / len(sentence_list1)\n",
    "\n",
    "    polarity_list.append(polarity)  \n",
    "    subjectivity_list.append(subjectivity)\n",
    "\n",
    "data4 = pd.DataFrame(polarity_list)  \n",
    "data5 = pd.DataFrame(subjectivity_list)  \n",
    "data6 = pd.merge(data4, data5, left_index=True, right_index=True)\n",
    "data6=data6.rename(columns={'0_x': \"Polarity\", '0_y': 'Subjectivity'})\n",
    "\n",
    "data6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...</td>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "      <td>-0.024999</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.388771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...</td>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "      <td>0.062156</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "      <td>0.086392</td>\n",
       "      <td>0.337145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...</td>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "      <td>-0.036351</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "      <td>0.019579</td>\n",
       "      <td>0.408200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...</td>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "      <td>0.008257</td>\n",
       "      <td>0.363186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...</td>\n",
       "      <td>A WARM WELCOME</td>\n",
       "      <td>0.095816</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "      <td>0.090141</td>\n",
       "      <td>0.430428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...</td>\n",
       "      <td>ON THE DOORSTEP</td>\n",
       "      <td>-0.004927</td>\n",
       "      <td>0.060587</td>\n",
       "      <td>0.888353</td>\n",
       "      <td>0.051073</td>\n",
       "      <td>-0.001852</td>\n",
       "      <td>0.360175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...</td>\n",
       "      <td>INSIDE INFORMATION</td>\n",
       "      <td>0.068378</td>\n",
       "      <td>0.082650</td>\n",
       "      <td>0.816149</td>\n",
       "      <td>0.101219</td>\n",
       "      <td>0.066962</td>\n",
       "      <td>0.379169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...</td>\n",
       "      <td>NOT AT HOME</td>\n",
       "      <td>-0.006529</td>\n",
       "      <td>0.057590</td>\n",
       "      <td>0.891776</td>\n",
       "      <td>0.050610</td>\n",
       "      <td>0.045913</td>\n",
       "      <td>0.357767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...</td>\n",
       "      <td>FIRE AND WATER</td>\n",
       "      <td>0.016736</td>\n",
       "      <td>0.071964</td>\n",
       "      <td>0.853494</td>\n",
       "      <td>0.074536</td>\n",
       "      <td>0.060123</td>\n",
       "      <td>0.349235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...</td>\n",
       "      <td>THE GATHERING OF THE CLOUDS</td>\n",
       "      <td>0.024960</td>\n",
       "      <td>0.081815</td>\n",
       "      <td>0.840382</td>\n",
       "      <td>0.077790</td>\n",
       "      <td>0.036574</td>\n",
       "      <td>0.364356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...</td>\n",
       "      <td>A THIEF IN THE NIGHT</td>\n",
       "      <td>0.070048</td>\n",
       "      <td>0.063877</td>\n",
       "      <td>0.856475</td>\n",
       "      <td>0.079656</td>\n",
       "      <td>0.041592</td>\n",
       "      <td>0.368692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...</td>\n",
       "      <td>THE CLOUDS BURST</td>\n",
       "      <td>-0.067713</td>\n",
       "      <td>0.096757</td>\n",
       "      <td>0.842087</td>\n",
       "      <td>0.061148</td>\n",
       "      <td>0.011715</td>\n",
       "      <td>0.288025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...</td>\n",
       "      <td>THE RETURN JOURNEY</td>\n",
       "      <td>0.083252</td>\n",
       "      <td>0.065162</td>\n",
       "      <td>0.832234</td>\n",
       "      <td>0.102604</td>\n",
       "      <td>0.068537</td>\n",
       "      <td>0.333555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...</td>\n",
       "      <td>THE LAST STAGE</td>\n",
       "      <td>0.164969</td>\n",
       "      <td>0.044680</td>\n",
       "      <td>0.857906</td>\n",
       "      <td>0.097422</td>\n",
       "      <td>0.091542</td>\n",
       "      <td>0.298514</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text                             Chapters  Compound  Negative   Neutral  Positive  Polarity  Subjectivity\n",
       "0   Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...                  AN UNEXPECTED PARTY  0.125155  0.046048  0.858940  0.095017  0.061535      0.391206\n",
       "1   Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...                         ROAST MUTTON  0.004399  0.063258  0.874296  0.062453  0.047677      0.319128\n",
       "2   Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...                         A SHORT REST  0.136103  0.035789  0.849283  0.114947  0.098664      0.358846\n",
       "3   Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...             OVER HILL AND UNDER HILL -0.034112  0.082478  0.848184  0.069357  0.004561      0.389319\n",
       "4   Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...                  RIDDLES IN THE DARK -0.043253  0.103889  0.824226  0.071892 -0.006902      0.339751\n",
       "5   Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...  OUT OF THE FRYING-PAN INTO THE FIRE -0.024999  0.079857  0.850042  0.070087  0.020121      0.388771\n",
       "6   Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...                       QUEER LODGINGS  0.062156  0.055134  0.858517  0.077430  0.086392      0.337145\n",
       "7   Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...                    FLIES AND SPIDERS -0.036351  0.087934  0.843843  0.068238  0.019579      0.408200\n",
       "8   Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...                  BARRELS OUT OF BOND  0.019908  0.075414  0.853004  0.071605  0.008257      0.363186\n",
       "9   Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...                       A WARM WELCOME  0.095816  0.057529  0.862180  0.080308  0.090141      0.430428\n",
       "10  Chapter XI \\n\\n\\nON THE DOORSTEP \\n\\n\\nIn two ...                      ON THE DOORSTEP -0.004927  0.060587  0.888353  0.051073 -0.001852      0.360175\n",
       "11  Chapter XII \\n\\n\\nINSIDE INFORMATION \\n\\n\\nFor...                   INSIDE INFORMATION  0.068378  0.082650  0.816149  0.101219  0.066962      0.379169\n",
       "12  Chapter X III \\n\\n\\nNOT AT HOME \\n\\n\\nIn the m...                          NOT AT HOME -0.006529  0.057590  0.891776  0.050610  0.045913      0.357767\n",
       "13  Chapter XIV \\n\\n\\nFIRE AND WATER \\n\\n\\nNow if ...                       FIRE AND WATER  0.016736  0.071964  0.853494  0.074536  0.060123      0.349235\n",
       "14  Chapter XV \\n\\n\\nTHE GATHERING OF THE \\n\\nCLOU...          THE GATHERING OF THE CLOUDS  0.024960  0.081815  0.840382  0.077790  0.036574      0.364356\n",
       "15  Chapter XVI \\n\\n\\nA THIEF IN THE NIGHT \\n\\n\\nN...                 A THIEF IN THE NIGHT  0.070048  0.063877  0.856475  0.079656  0.041592      0.368692\n",
       "16  Chapter XVII \\n\\n\\nTHE CLOUDS BURST \\n\\n\\nNext...                     THE CLOUDS BURST -0.067713  0.096757  0.842087  0.061148  0.011715      0.288025\n",
       "17  Chapter XVIII \\n\\n\\nTHE RETURN JOURNEY \\n\\n\\nW...                   THE RETURN JOURNEY  0.083252  0.065162  0.832234  0.102604  0.068537      0.333555\n",
       "18  Chapter X IX \\n\\n\\nTHE LAST STAGE \\n\\n\\nIt was...                       THE LAST STAGE  0.164969  0.044680  0.857906  0.097422  0.091542      0.298514"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = pd.merge(Hobbit_sentiments, data6, left_index=True, right_index=True)\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>36814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>55336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>31430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>21476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>16375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>38863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>21554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>17788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>18278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>11453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>21772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>15411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>13639</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Total_words\n",
       "0         47620\n",
       "1         28863\n",
       "2         15671\n",
       "3         22684\n",
       "4         38059\n",
       "5         36814\n",
       "6         48561\n",
       "7         55336\n",
       "8         31430\n",
       "9         21476\n",
       "10        16375\n",
       "11        38863\n",
       "12        21554\n",
       "13        17788\n",
       "14        18278\n",
       "15        11453\n",
       "16        21772\n",
       "17        15411\n",
       "18        13639"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chapters = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,ch18,ch19]\n",
    "\n",
    "wordlength_list = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    count = len(chapter)\n",
    "        \n",
    "    wordlength_list.append(count)  \n",
    "\n",
    "data7 = pd.DataFrame(wordlength_list)  \n",
    "data7=data7.rename(columns={0: 'Total_words'})\n",
    "data7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Total_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>47620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>28863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>15671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>22684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>38059</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text                  Chapters  Compound  Negative   Neutral  Positive  Polarity  Subjectivity  Total_words\n",
       "0  Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...       AN UNEXPECTED PARTY  0.125155  0.046048  0.858940  0.095017  0.061535      0.391206        47620\n",
       "1  Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...              ROAST MUTTON  0.004399  0.063258  0.874296  0.062453  0.047677      0.319128        28863\n",
       "2  Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...              A SHORT REST  0.136103  0.035789  0.849283  0.114947  0.098664      0.358846        15671\n",
       "3  Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...  OVER HILL AND UNDER HILL -0.034112  0.082478  0.848184  0.069357  0.004561      0.389319        22684\n",
       "4  Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...       RIDDLES IN THE DARK -0.043253  0.103889  0.824226  0.071892 -0.006902      0.339751        38059"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = pd.merge(Hobbit_sentiments, data7, left_index=True, right_index=True)\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>47620</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>28863</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>15671</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>22684</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>38059</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...</td>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "      <td>-0.024999</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.388771</td>\n",
       "      <td>36814</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...</td>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "      <td>0.062156</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "      <td>0.086392</td>\n",
       "      <td>0.337145</td>\n",
       "      <td>48561</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...</td>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "      <td>-0.036351</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "      <td>0.019579</td>\n",
       "      <td>0.408200</td>\n",
       "      <td>55336</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...</td>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "      <td>0.008257</td>\n",
       "      <td>0.363186</td>\n",
       "      <td>31430</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...</td>\n",
       "      <td>A WARM WELCOME</td>\n",
       "      <td>0.095816</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "      <td>0.090141</td>\n",
       "      <td>0.430428</td>\n",
       "      <td>21476</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text                             Chapters  Compound  Negative   Neutral  Positive  Polarity  Subjectivity  Total_words    Rating\n",
       "0  Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...                  AN UNEXPECTED PARTY  0.125155  0.046048  0.858940  0.095017  0.061535      0.391206        47620  Positive\n",
       "1  Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...                         ROAST MUTTON  0.004399  0.063258  0.874296  0.062453  0.047677      0.319128        28863   Neutral\n",
       "2  Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...                         A SHORT REST  0.136103  0.035789  0.849283  0.114947  0.098664      0.358846        15671  Positive\n",
       "3  Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...             OVER HILL AND UNDER HILL -0.034112  0.082478  0.848184  0.069357  0.004561      0.389319        22684   Neutral\n",
       "4  Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...                  RIDDLES IN THE DARK -0.043253  0.103889  0.824226  0.071892 -0.006902      0.339751        38059   Neutral\n",
       "5  Chapter VI \\n\\n\\nOUT OF THE FRYING-PAN INTO \\n...  OUT OF THE FRYING-PAN INTO THE FIRE -0.024999  0.079857  0.850042  0.070087  0.020121      0.388771        36814   Neutral\n",
       "6  Chapter VII \\n\\n\\nQUEER LODGINGS \\n\\n\\nThe nex...                       QUEER LODGINGS  0.062156  0.055134  0.858517  0.077430  0.086392      0.337145        48561  Positive\n",
       "7  Chapter VIII \\n\\n\\nFLIES AND SPIDERS \\n\\n\\nThe...                    FLIES AND SPIDERS -0.036351  0.087934  0.843843  0.068238  0.019579      0.408200        55336   Neutral\n",
       "8  Chapter IX \\n\\n\\nBARRELS OUT OF BOND \\n\\n\\nThe...                  BARRELS OUT OF BOND  0.019908  0.075414  0.853004  0.071605  0.008257      0.363186        31430   Neutral\n",
       "9  Chapter X \\n\\n\\nA WARM WELCOME \\n\\n\\nThe day g...                       A WARM WELCOME  0.095816  0.057529  0.862180  0.080308  0.090141      0.430428        21476  Positive"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def rating(Hobbit_sentiments):\n",
    "    if Hobbit_sentiments['Compound'] > 0.05:\n",
    "        return 'Positive'\n",
    "    elif Hobbit_sentiments['Compound'] < -0.05:\n",
    "        return 'Negative'\n",
    "    else:\n",
    "        return 'Neutral'\n",
    "\n",
    "Hobbit_sentiments['Rating'] = Hobbit_sentiments.apply(rating, axis=1)\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments.head(19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('hobbit', 30),\n",
       "  ('door', 29),\n",
       "  ('bilbo', 24),\n",
       "  ('time', 21),\n",
       "  ('gandalf', 21),\n",
       "  ('thorin', 18),\n",
       "  ('morning', 16),\n",
       "  ('day', 12),\n",
       "  ('dragon', 12),\n",
       "  ('mountain', 12)],\n",
       " [('bilbo', 24),\n",
       "  ('time', 15),\n",
       "  ('william', 14),\n",
       "  ('fire', 13),\n",
       "  ('bert', 13),\n",
       "  ('night', 11),\n",
       "  ('gandalf', 10),\n",
       "  ('pocket', 10),\n",
       "  ('way', 9),\n",
       "  ('yer', 9)],\n",
       " [('day', 13),\n",
       "  ('house', 10),\n",
       "  ('moon', 10),\n",
       "  ('water', 8),\n",
       "  ('way', 6),\n",
       "  ('time', 6),\n",
       "  ('elrond', 6),\n",
       "  ('bilbo', 6),\n",
       "  ('morning', 5),\n",
       "  ('thing', 5)],\n",
       " [('bilbo', 16),\n",
       "  ('gandalf', 15),\n",
       "  ('goblin', 12),\n",
       "  ('way', 9),\n",
       "  ('sword', 9),\n",
       "  ('crack', 7),\n",
       "  ('corner', 7),\n",
       "  ('time', 7),\n",
       "  ('mountain', 6),\n",
       "  ('nothing', 6)],\n",
       " [('gollum', 66),\n",
       "  ('bilbo', 51),\n",
       "  ('way', 24),\n",
       "  ('time', 20),\n",
       "  ('door', 13),\n",
       "  ('thing', 12),\n",
       "  ('water', 10),\n",
       "  ('passage', 10),\n",
       "  ('nothing', 9),\n",
       "  ('hand', 9)],\n",
       " [('gandalf', 19),\n",
       "  ('bilbo', 18),\n",
       "  ('fire', 17),\n",
       "  ('time', 13),\n",
       "  ('night', 11),\n",
       "  ('path', 9),\n",
       "  ('dori', 9),\n",
       "  ('way', 9),\n",
       "  ('round', 9),\n",
       "  ('lord', 9)],\n",
       " [('gandalf', 21),\n",
       "  ('way', 19),\n",
       "  ('time', 17),\n",
       "  ('bilbo', 15),\n",
       "  ('house', 15),\n",
       "  ('night', 14),\n",
       "  ('hobbit', 12),\n",
       "  ('carrock', 12),\n",
       "  ('mirkwood', 12),\n",
       "  ('day', 11)],\n",
       " [('bilbo', 35),\n",
       "  ('time', 31),\n",
       "  ('path', 19),\n",
       "  ('way', 18),\n",
       "  ('bombur', 18),\n",
       "  ('hobbit', 14),\n",
       "  ('spider', 14),\n",
       "  ('rope', 13),\n",
       "  ('dark', 12),\n",
       "  ('nothing', 12)],\n",
       " [('bilbo', 25),\n",
       "  ('water', 24),\n",
       "  ('time', 22),\n",
       "  ('way', 10),\n",
       "  ('river', 10),\n",
       "  ('day', 9),\n",
       "  ('barrel', 9),\n",
       "  ('hobbit', 8),\n",
       "  ('gate', 8),\n",
       "  ('stream', 8)],\n",
       " [('town', 24),\n",
       "  ('mountain', 19),\n",
       "  ('river', 16),\n",
       "  ('bilbo', 9),\n",
       "  ('master', 9),\n",
       "  ('wood', 8),\n",
       "  ('thorin', 8),\n",
       "  ('end', 7),\n",
       "  ('thror', 7),\n",
       "  ('day', 6)],\n",
       " [('day', 12),\n",
       "  ('mountain', 11),\n",
       "  ('wall', 9),\n",
       "  ('door', 8),\n",
       "  ('valley', 8),\n",
       "  ('rock', 8),\n",
       "  ('end', 7),\n",
       "  ('bilbo', 7),\n",
       "  ('doorstep', 6),\n",
       "  ('stone', 6)],\n",
       " [('dragon', 25),\n",
       "  ('smaug', 24),\n",
       "  ('bilbo', 19),\n",
       "  ('door', 18),\n",
       "  ('mountain', 18),\n",
       "  ('tunnel', 17),\n",
       "  ('time', 15),\n",
       "  ('hobbit', 14),\n",
       "  ('end', 10),\n",
       "  ('way', 9)],\n",
       " [('bilbo', 15),\n",
       "  ('door', 10),\n",
       "  ('light', 9),\n",
       "  ('mountain', 8),\n",
       "  ('time', 7),\n",
       "  ('dragon', 7),\n",
       "  ('torch', 7),\n",
       "  ('end', 6),\n",
       "  ('way', 6),\n",
       "  ('floor', 6)],\n",
       " [('town', 19),\n",
       "  ('dragon', 14),\n",
       "  ('fire', 10),\n",
       "  ('water', 10),\n",
       "  ('mountain', 10),\n",
       "  ('master', 10),\n",
       "  ('lake', 8),\n",
       "  ('bard', 8),\n",
       "  ('king', 7),\n",
       "  ('time', 6)],\n",
       " [('mountain', 16),\n",
       "  ('thorin', 9),\n",
       "  ('bilbo', 7),\n",
       "  ('time', 7),\n",
       "  ('dragon', 7),\n",
       "  ('wall', 7),\n",
       "  ('treasure', 6),\n",
       "  ('gate', 6),\n",
       "  ('return', 5),\n",
       "  ('battle', 5)],\n",
       " [('time', 10),\n",
       "  ('bilbo', 9),\n",
       "  ('thorin', 6),\n",
       "  ('stone', 5),\n",
       "  ('midnight', 5),\n",
       "  ('wall', 4),\n",
       "  ('hobbit', 4),\n",
       "  ('bard', 4),\n",
       "  ('night', 3),\n",
       "  ('dain', 3)],\n",
       " [('mountain', 18),\n",
       "  ('dain', 10),\n",
       "  ('bilbo', 9),\n",
       "  ('thorin', 8),\n",
       "  ('bard', 8),\n",
       "  ('battle', 8),\n",
       "  ('way', 7),\n",
       "  ('gandalf', 7),\n",
       "  ('time', 7),\n",
       "  ('side', 7)],\n",
       " [('bilbo', 19),\n",
       "  ('farewell', 7),\n",
       "  ('mountain', 7),\n",
       "  ('way', 7),\n",
       "  ('man', 6),\n",
       "  ('voice', 6),\n",
       "  ('thorin', 6),\n",
       "  ('gandalf', 6),\n",
       "  ('gold', 6),\n",
       "  ('time', 5)],\n",
       " [('bilbo', 12),\n",
       "  ('time', 9),\n",
       "  ('gold', 7),\n",
       "  ('la', 6),\n",
       "  ('elrond', 6),\n",
       "  ('valley', 5),\n",
       "  ('house', 5),\n",
       "  ('water', 5),\n",
       "  ('moon', 5),\n",
       "  ('river', 5)]]"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import io, re, nltk\n",
    "from pathlib import Path\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import FreqDist\n",
    "\n",
    "chapters = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,ch18,ch19]\n",
    "\n",
    "Noun_freq = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    stop = set(stopwords.words('english'))\n",
    "    words = [word for word in tokens if word not in stop]\n",
    "    tagged = pos_tag(words)\n",
    "    nouns = [word for word, pos in tagged if (pos == 'NN')]\n",
    "    Noun1 = (0)\n",
    "            \n",
    "    Noun1 = FreqDist(nouns).most_common(10)\n",
    "    \n",
    "    Noun_freq.append(Noun1)\n",
    "    \n",
    "Noun_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lex_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.190665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.229236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.275179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.256335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.181741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.200706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.173946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.171966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.214007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.263410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.281662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.221792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.260661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.297677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.272647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.304147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.257935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.293329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.319428</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Lex_density\n",
       "0      0.190665\n",
       "1      0.229236\n",
       "2      0.275179\n",
       "3      0.256335\n",
       "4      0.181741\n",
       "5      0.200706\n",
       "6      0.173946\n",
       "7      0.171966\n",
       "8      0.214007\n",
       "9      0.263410\n",
       "10     0.281662\n",
       "11     0.221792\n",
       "12     0.260661\n",
       "13     0.297677\n",
       "14     0.272647\n",
       "15     0.304147\n",
       "16     0.257935\n",
       "17     0.293329\n",
       "18     0.319428"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Lexical Diversity represents the ratio of unique words used to the total number of words in the story.\n",
    "\n",
    "chapters = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,ch18,ch19]\n",
    "\n",
    "def lexical_density(text):\n",
    "    return len(set(text)) / len(text)\n",
    "\n",
    "Lexical_density = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    LexDen = (0)\n",
    "            \n",
    "    LexDen = lexical_density(tokens)\n",
    "\n",
    "    Lexical_density.append(LexDen)  \n",
    "\n",
    "LexDen = pd.DataFrame(Lexical_density) \n",
    "LexDen=LexDen.rename(columns={0: 'Lex_density'})\n",
    "LexDen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lex_density_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.190665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.229236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.275179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.256335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.181741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.200706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.173946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.171966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.214007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.263410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.281662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.221792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.260661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.297677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.272647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.304147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.257935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.293329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.319428</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Lex_density_norm\n",
       "0           0.190665\n",
       "1           0.229236\n",
       "2           0.275179\n",
       "3           0.256335\n",
       "4           0.181741\n",
       "5           0.200706\n",
       "6           0.173946\n",
       "7           0.171966\n",
       "8           0.214007\n",
       "9           0.263410\n",
       "10          0.281662\n",
       "11          0.221792\n",
       "12          0.260661\n",
       "13          0.297677\n",
       "14          0.272647\n",
       "15          0.304147\n",
       "16          0.257935\n",
       "17          0.293329\n",
       "18          0.319428"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Lexical Diversity represents the ratio of unique words used to the total number of words in the story.\n",
    "\n",
    "chapters = [ch1,ch2,ch3,ch4,ch5,ch6,ch7,ch8,ch9,ch10,ch11,ch12,ch13,ch14,ch15,ch16,ch17,ch18,ch19]\n",
    "\n",
    "def lexical_density(text):\n",
    "    return len(set(text)) / len(text)\n",
    "\n",
    "Lexical_density_norm = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    tokens = tokens[0:11453]\n",
    "    LexDen1 = (0)\n",
    "            \n",
    "    LexDen1 = lexical_density(tokens)\n",
    "\n",
    "    Lexical_density_norm.append(LexDen1)  \n",
    "\n",
    "LexDen_norm = pd.DataFrame(Lexical_density_norm) \n",
    "LexDen_norm=LexDen_norm.rename(columns={0: 'Lex_density_norm'})\n",
    "LexDen_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Chapters                                               Text  Total_words  Lex_density  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0       AN UNEXPECTED PARTY  Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...        47620     0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206\n",
       "1              ROAST MUTTON  Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...        28863     0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128\n",
       "2              A SHORT REST  Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...        15671     0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846\n",
       "3  OVER HILL AND UNDER HILL  Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...        22684     0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319\n",
       "4       RIDDLES IN THE DARK  Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...        38059     0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = pd.merge(Hobbit_sentiments, LexDen, left_index=True, right_index=True)\n",
    "\n",
    "Hobbit_sentiments = Hobbit_sentiments[['Chapters','Text','Total_words','Lex_density','Negative',\n",
    "                                                  'Neutral','Positive','Compound','Rating','Polarity','Subjectivity']]\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0       AN UNEXPECTED PARTY  Chapter I \\n\\n\\n\\nAN UNEXPECTED PARTY \\n\\n\\nIn...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206\n",
       "1              ROAST MUTTON  Chapter II \\n\\n\\nROAST MUTTON \\n\\n\\nUp jumped ...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128\n",
       "2              A SHORT REST  Chapter III \\n\\n\\nA SHORT REST \\n\\n\\nThey did ...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846\n",
       "3  OVER HILL AND UNDER HILL  Chapter IV \\n\\n\\nOVER HILL AND UNDER HILL \\n\\n...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319\n",
       "4       RIDDLES IN THE DARK  Chapter V \\n\\n\\nRIDDLES IN THE DARK \\n\\n\\nWhen...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = pd.merge(Hobbit_sentiments, LexDen_norm, left_index=True, right_index=True)\n",
    "\n",
    "Hobbit_sentiments = Hobbit_sentiments[['Chapters','Text','Total_words','Lex_density','Lex_density_norm','Negative',\n",
    "                                                  'Neutral','Positive','Compound','Rating','Polarity','Subjectivity']]\n",
    "\n",
    "Hobbit_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I     AN UNEXPECTED PARTY    In a hole...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II    ROAST MUTTON    Up jumped Bilbo,...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III    A SHORT REST    They did not si...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV    OVER HILL AND UNDER HILL    Ther...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V    RIDDLES IN THE DARK    When Bilbo...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "      <td>Chapter VI    OUT OF THE FRYING-PAN INTO   THE...</td>\n",
       "      <td>36814</td>\n",
       "      <td>0.200706</td>\n",
       "      <td>0.200706</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "      <td>-0.024999</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.388771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "      <td>Chapter VII    QUEER LODGINGS    The next morn...</td>\n",
       "      <td>48561</td>\n",
       "      <td>0.173946</td>\n",
       "      <td>0.173946</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "      <td>0.062156</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.086392</td>\n",
       "      <td>0.337145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "      <td>Chapter VIII    FLIES AND SPIDERS    They walk...</td>\n",
       "      <td>55336</td>\n",
       "      <td>0.171966</td>\n",
       "      <td>0.171966</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "      <td>-0.036351</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.019579</td>\n",
       "      <td>0.408200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "      <td>Chapter IX    BARRELS OUT OF BOND    The day a...</td>\n",
       "      <td>31430</td>\n",
       "      <td>0.214007</td>\n",
       "      <td>0.214007</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.008257</td>\n",
       "      <td>0.363186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>A WARM WELCOME</td>\n",
       "      <td>Chapter X    A WARM WELCOME    The day grew li...</td>\n",
       "      <td>21476</td>\n",
       "      <td>0.263410</td>\n",
       "      <td>0.263410</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "      <td>0.095816</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.090141</td>\n",
       "      <td>0.430428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ON THE DOORSTEP</td>\n",
       "      <td>Chapter XI    ON THE DOORSTEP    In two days g...</td>\n",
       "      <td>16375</td>\n",
       "      <td>0.281662</td>\n",
       "      <td>0.281662</td>\n",
       "      <td>0.060587</td>\n",
       "      <td>0.888353</td>\n",
       "      <td>0.051073</td>\n",
       "      <td>-0.004927</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.001852</td>\n",
       "      <td>0.360175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>INSIDE INFORMATION</td>\n",
       "      <td>Chapter XII    INSIDE INFORMATION    For a lon...</td>\n",
       "      <td>38863</td>\n",
       "      <td>0.221792</td>\n",
       "      <td>0.221792</td>\n",
       "      <td>0.082650</td>\n",
       "      <td>0.816149</td>\n",
       "      <td>0.101219</td>\n",
       "      <td>0.068378</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.066962</td>\n",
       "      <td>0.379169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NOT AT HOME</td>\n",
       "      <td>Chapter X III    NOT AT HOME    In the meanwhi...</td>\n",
       "      <td>21554</td>\n",
       "      <td>0.260661</td>\n",
       "      <td>0.260661</td>\n",
       "      <td>0.057590</td>\n",
       "      <td>0.891776</td>\n",
       "      <td>0.050610</td>\n",
       "      <td>-0.006529</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.045913</td>\n",
       "      <td>0.357767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FIRE AND WATER</td>\n",
       "      <td>Chapter XIV    FIRE AND WATER    Now if you wi...</td>\n",
       "      <td>17788</td>\n",
       "      <td>0.297677</td>\n",
       "      <td>0.297677</td>\n",
       "      <td>0.071964</td>\n",
       "      <td>0.853494</td>\n",
       "      <td>0.074536</td>\n",
       "      <td>0.016736</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.060123</td>\n",
       "      <td>0.349235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>THE GATHERING OF THE CLOUDS</td>\n",
       "      <td>Chapter XV    THE GATHERING OF THE   CLOUDS   ...</td>\n",
       "      <td>18278</td>\n",
       "      <td>0.272647</td>\n",
       "      <td>0.272647</td>\n",
       "      <td>0.081815</td>\n",
       "      <td>0.840382</td>\n",
       "      <td>0.077790</td>\n",
       "      <td>0.024960</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.036574</td>\n",
       "      <td>0.364356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>A THIEF IN THE NIGHT</td>\n",
       "      <td>Chapter XVI    A THIEF IN THE NIGHT    Now the...</td>\n",
       "      <td>11453</td>\n",
       "      <td>0.304147</td>\n",
       "      <td>0.304147</td>\n",
       "      <td>0.063877</td>\n",
       "      <td>0.856475</td>\n",
       "      <td>0.079656</td>\n",
       "      <td>0.070048</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041592</td>\n",
       "      <td>0.368692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>THE CLOUDS BURST</td>\n",
       "      <td>Chapter XVII    THE CLOUDS BURST    Next day t...</td>\n",
       "      <td>21772</td>\n",
       "      <td>0.257935</td>\n",
       "      <td>0.257935</td>\n",
       "      <td>0.096757</td>\n",
       "      <td>0.842087</td>\n",
       "      <td>0.061148</td>\n",
       "      <td>-0.067713</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0.011715</td>\n",
       "      <td>0.288025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>THE RETURN JOURNEY</td>\n",
       "      <td>Chapter XVIII    THE RETURN JOURNEY    When Bi...</td>\n",
       "      <td>15411</td>\n",
       "      <td>0.293329</td>\n",
       "      <td>0.293329</td>\n",
       "      <td>0.065162</td>\n",
       "      <td>0.832234</td>\n",
       "      <td>0.102604</td>\n",
       "      <td>0.083252</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.068537</td>\n",
       "      <td>0.333555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>THE LAST STAGE</td>\n",
       "      <td>Chapter X IX    THE LAST STAGE    It was on Ma...</td>\n",
       "      <td>13639</td>\n",
       "      <td>0.319428</td>\n",
       "      <td>0.319428</td>\n",
       "      <td>0.044680</td>\n",
       "      <td>0.857906</td>\n",
       "      <td>0.097422</td>\n",
       "      <td>0.164969</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.091542</td>\n",
       "      <td>0.298514</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0                   AN UNEXPECTED PARTY  Chapter I     AN UNEXPECTED PARTY    In a hole...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206\n",
       "1                          ROAST MUTTON  Chapter II    ROAST MUTTON    Up jumped Bilbo,...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128\n",
       "2                          A SHORT REST  Chapter III    A SHORT REST    They did not si...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846\n",
       "3              OVER HILL AND UNDER HILL  Chapter IV    OVER HILL AND UNDER HILL    Ther...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319\n",
       "4                   RIDDLES IN THE DARK  Chapter V    RIDDLES IN THE DARK    When Bilbo...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751\n",
       "5   OUT OF THE FRYING-PAN INTO THE FIRE  Chapter VI    OUT OF THE FRYING-PAN INTO   THE...        36814     0.200706          0.200706  0.079857  0.850042  0.070087 -0.024999   Neutral  0.020121      0.388771\n",
       "6                        QUEER LODGINGS  Chapter VII    QUEER LODGINGS    The next morn...        48561     0.173946          0.173946  0.055134  0.858517  0.077430  0.062156  Positive  0.086392      0.337145\n",
       "7                     FLIES AND SPIDERS  Chapter VIII    FLIES AND SPIDERS    They walk...        55336     0.171966          0.171966  0.087934  0.843843  0.068238 -0.036351   Neutral  0.019579      0.408200\n",
       "8                   BARRELS OUT OF BOND  Chapter IX    BARRELS OUT OF BOND    The day a...        31430     0.214007          0.214007  0.075414  0.853004  0.071605  0.019908   Neutral  0.008257      0.363186\n",
       "9                        A WARM WELCOME  Chapter X    A WARM WELCOME    The day grew li...        21476     0.263410          0.263410  0.057529  0.862180  0.080308  0.095816  Positive  0.090141      0.430428\n",
       "10                      ON THE DOORSTEP  Chapter XI    ON THE DOORSTEP    In two days g...        16375     0.281662          0.281662  0.060587  0.888353  0.051073 -0.004927   Neutral -0.001852      0.360175\n",
       "11                   INSIDE INFORMATION  Chapter XII    INSIDE INFORMATION    For a lon...        38863     0.221792          0.221792  0.082650  0.816149  0.101219  0.068378  Positive  0.066962      0.379169\n",
       "12                          NOT AT HOME  Chapter X III    NOT AT HOME    In the meanwhi...        21554     0.260661          0.260661  0.057590  0.891776  0.050610 -0.006529   Neutral  0.045913      0.357767\n",
       "13                       FIRE AND WATER  Chapter XIV    FIRE AND WATER    Now if you wi...        17788     0.297677          0.297677  0.071964  0.853494  0.074536  0.016736   Neutral  0.060123      0.349235\n",
       "14          THE GATHERING OF THE CLOUDS  Chapter XV    THE GATHERING OF THE   CLOUDS   ...        18278     0.272647          0.272647  0.081815  0.840382  0.077790  0.024960   Neutral  0.036574      0.364356\n",
       "15                 A THIEF IN THE NIGHT  Chapter XVI    A THIEF IN THE NIGHT    Now the...        11453     0.304147          0.304147  0.063877  0.856475  0.079656  0.070048  Positive  0.041592      0.368692\n",
       "16                     THE CLOUDS BURST  Chapter XVII    THE CLOUDS BURST    Next day t...        21772     0.257935          0.257935  0.096757  0.842087  0.061148 -0.067713  Negative  0.011715      0.288025\n",
       "17                   THE RETURN JOURNEY  Chapter XVIII    THE RETURN JOURNEY    When Bi...        15411     0.293329          0.293329  0.065162  0.832234  0.102604  0.083252  Positive  0.068537      0.333555\n",
       "18                       THE LAST STAGE  Chapter X IX    THE LAST STAGE    It was on Ma...        13639     0.319428          0.319428  0.044680  0.857906  0.097422  0.164969  Positive  0.091542      0.298514"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments = Hobbit_sentiments.replace('\\n',' ', regex=True) \n",
    "Hobbit_sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from tqdm import trange\n",
    "\n",
    "\n",
    "def text_emotion(df, column):\n",
    "    '''\n",
    "    INPUT: DataFrame, string\n",
    "    OUTPUT: the original DataFrame with ten new columns for each emotion\n",
    "    '''\n",
    "\n",
    "    new_df = df.copy()\n",
    "\n",
    "    xlsx = pd.read_excel(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Silmarillion\\NRC-Sentiment-Emotion-Lexicons\\NRC-Sentiment-Emotion-Lexicons\\NRC-Emotion-Lexicon-v0.92\\NRC-Emotion-Lexicon-v0.92-In105Languages-Nov2017Translations.xlsx')\n",
    "    emolex_df = xlsx[['word', 'Positive_NRC','Negative_NRC','Anger', 'Anticipation', 'Disgust', 'Fear','Joy',\n",
    "                      'Sadness', 'Surprise', 'Trust']]\n",
    "    emotions = emolex_df.columns.drop('word')\n",
    "    emo_df = pd.DataFrame(0, index=df.index, columns=emotions)\n",
    "\n",
    "    stemmer = SnowballStemmer(\"english\")\n",
    "\n",
    "    \n",
    "    with tqdm(total=len(list(new_df.iterrows()))) as pbar:\n",
    "        for i, row in new_df.iterrows():\n",
    "            pbar.update(1)\n",
    "            document = word_tokenize(new_df.loc[i][column])\n",
    "            for word in document:\n",
    "                word = stemmer.stem(word.lower())\n",
    "                emo_score = emolex_df[emolex_df.word == word]\n",
    "                if not emo_score.empty:\n",
    "                    for emotion in list(emotions):\n",
    "                        emo_df.at[i, emotion] += emo_score[emotion]\n",
    "\n",
    "    new_df = pd.concat([new_df, emo_df], axis=1)\n",
    "\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e787c7dd48841ec909d0806059308d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=19), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "Hobbit_sentiments_final = text_emotion(Hobbit_sentiments, 'Text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I     AN UNEXPECTED PARTY    In a hole...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>342</td>\n",
       "      <td>213</td>\n",
       "      <td>82</td>\n",
       "      <td>241</td>\n",
       "      <td>69</td>\n",
       "      <td>152</td>\n",
       "      <td>159</td>\n",
       "      <td>119</td>\n",
       "      <td>115</td>\n",
       "      <td>226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II    ROAST MUTTON    Up jumped Bilbo,...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>164</td>\n",
       "      <td>164</td>\n",
       "      <td>97</td>\n",
       "      <td>119</td>\n",
       "      <td>54</td>\n",
       "      <td>98</td>\n",
       "      <td>86</td>\n",
       "      <td>42</td>\n",
       "      <td>66</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III    A SHORT REST    They did not si...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>134</td>\n",
       "      <td>61</td>\n",
       "      <td>34</td>\n",
       "      <td>88</td>\n",
       "      <td>28</td>\n",
       "      <td>34</td>\n",
       "      <td>71</td>\n",
       "      <td>34</td>\n",
       "      <td>47</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV    OVER HILL AND UNDER HILL    Ther...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>134</td>\n",
       "      <td>226</td>\n",
       "      <td>95</td>\n",
       "      <td>97</td>\n",
       "      <td>80</td>\n",
       "      <td>142</td>\n",
       "      <td>59</td>\n",
       "      <td>85</td>\n",
       "      <td>69</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V    RIDDLES IN THE DARK    When Bilbo...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>195</td>\n",
       "      <td>307</td>\n",
       "      <td>128</td>\n",
       "      <td>172</td>\n",
       "      <td>76</td>\n",
       "      <td>160</td>\n",
       "      <td>97</td>\n",
       "      <td>127</td>\n",
       "      <td>162</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC  Anger  Anticipation  Disgust  Fear  Joy  Sadness  Surprise  Trust\n",
       "0       AN UNEXPECTED PARTY  Chapter I     AN UNEXPECTED PARTY    In a hole...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206           342           213     82           241       69   152  159      119       115    226\n",
       "1              ROAST MUTTON  Chapter II    ROAST MUTTON    Up jumped Bilbo,...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128           164           164     97           119       54    98   86       42        66    103\n",
       "2              A SHORT REST  Chapter III    A SHORT REST    They did not si...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846           134            61     34            88       28    34   71       34        47     81\n",
       "3  OVER HILL AND UNDER HILL  Chapter IV    OVER HILL AND UNDER HILL    Ther...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319           134           226     95            97       80   142   59       85        69     77\n",
       "4       RIDDLES IN THE DARK  Chapter V    RIDDLES IN THE DARK    When Bilbo...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751           195           307    128           172       76   160   97      127       162    102"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I     AN UNEXPECTED PARTY    In a hole...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>342</td>\n",
       "      <td>213</td>\n",
       "      <td>82</td>\n",
       "      <td>241</td>\n",
       "      <td>69</td>\n",
       "      <td>152</td>\n",
       "      <td>159</td>\n",
       "      <td>119</td>\n",
       "      <td>115</td>\n",
       "      <td>226</td>\n",
       "      <td>10400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II    ROAST MUTTON    Up jumped Bilbo,...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>164</td>\n",
       "      <td>164</td>\n",
       "      <td>97</td>\n",
       "      <td>119</td>\n",
       "      <td>54</td>\n",
       "      <td>98</td>\n",
       "      <td>86</td>\n",
       "      <td>42</td>\n",
       "      <td>66</td>\n",
       "      <td>103</td>\n",
       "      <td>6455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III    A SHORT REST    They did not si...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>134</td>\n",
       "      <td>61</td>\n",
       "      <td>34</td>\n",
       "      <td>88</td>\n",
       "      <td>28</td>\n",
       "      <td>34</td>\n",
       "      <td>71</td>\n",
       "      <td>34</td>\n",
       "      <td>47</td>\n",
       "      <td>81</td>\n",
       "      <td>3394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV    OVER HILL AND UNDER HILL    Ther...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>134</td>\n",
       "      <td>226</td>\n",
       "      <td>95</td>\n",
       "      <td>97</td>\n",
       "      <td>80</td>\n",
       "      <td>142</td>\n",
       "      <td>59</td>\n",
       "      <td>85</td>\n",
       "      <td>69</td>\n",
       "      <td>77</td>\n",
       "      <td>4759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V    RIDDLES IN THE DARK    When Bilbo...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>195</td>\n",
       "      <td>307</td>\n",
       "      <td>128</td>\n",
       "      <td>172</td>\n",
       "      <td>76</td>\n",
       "      <td>160</td>\n",
       "      <td>97</td>\n",
       "      <td>127</td>\n",
       "      <td>162</td>\n",
       "      <td>102</td>\n",
       "      <td>8588</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC  Anger  Anticipation  Disgust  Fear  Joy  Sadness  Surprise  Trust  word_count\n",
       "0       AN UNEXPECTED PARTY  Chapter I     AN UNEXPECTED PARTY    In a hole...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206           342           213     82           241       69   152  159      119       115    226       10400\n",
       "1              ROAST MUTTON  Chapter II    ROAST MUTTON    Up jumped Bilbo,...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128           164           164     97           119       54    98   86       42        66    103        6455\n",
       "2              A SHORT REST  Chapter III    A SHORT REST    They did not si...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846           134            61     34            88       28    34   71       34        47     81        3394\n",
       "3  OVER HILL AND UNDER HILL  Chapter IV    OVER HILL AND UNDER HILL    Ther...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319           134           226     95            97       80   142   59       85        69     77        4759\n",
       "4       RIDDLES IN THE DARK  Chapter V    RIDDLES IN THE DARK    When Bilbo...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751           195           307    128           172       76   160   97      127       162    102        8588"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments_final['word_count'] = Hobbit_sentiments_final['Text'].apply(tokenize.word_tokenize).apply(len)\n",
    "Hobbit_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Chapters             object\n",
       "Text                 object\n",
       "Total_words           int64\n",
       "Lex_density         float64\n",
       "Lex_density_norm    float64\n",
       "Negative            float64\n",
       "Neutral             float64\n",
       "Positive            float64\n",
       "Compound            float64\n",
       "Rating               object\n",
       "Polarity            float64\n",
       "Subjectivity        float64\n",
       "Positive_NRC          int64\n",
       "Negative_NRC          int64\n",
       "Anger                 int64\n",
       "Anticipation          int64\n",
       "Disgust               int64\n",
       "Fear                  int64\n",
       "Joy                   int64\n",
       "Sadness               int64\n",
       "Surprise              int64\n",
       "Trust                 int64\n",
       "word_count            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Hobbit_sentiments_final.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions = ['Positive_NRC','Negative_NRC','Anger','Anticipation','Disgust','Fear','Joy','Sadness','Surprise',\n",
    "                 'Trust']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I     AN UNEXPECTED PARTY    In a hole...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>0.032885</td>\n",
       "      <td>0.020481</td>\n",
       "      <td>0.007885</td>\n",
       "      <td>0.023173</td>\n",
       "      <td>0.006635</td>\n",
       "      <td>0.014615</td>\n",
       "      <td>0.015288</td>\n",
       "      <td>0.011442</td>\n",
       "      <td>0.011058</td>\n",
       "      <td>0.021731</td>\n",
       "      <td>10400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II    ROAST MUTTON    Up jumped Bilbo,...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>0.025407</td>\n",
       "      <td>0.025407</td>\n",
       "      <td>0.015027</td>\n",
       "      <td>0.018435</td>\n",
       "      <td>0.008366</td>\n",
       "      <td>0.015182</td>\n",
       "      <td>0.013323</td>\n",
       "      <td>0.006507</td>\n",
       "      <td>0.010225</td>\n",
       "      <td>0.015957</td>\n",
       "      <td>6455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III    A SHORT REST    They did not si...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>0.039481</td>\n",
       "      <td>0.017973</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.025928</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.020919</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.013848</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>3394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV    OVER HILL AND UNDER HILL    Ther...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>0.028157</td>\n",
       "      <td>0.047489</td>\n",
       "      <td>0.019962</td>\n",
       "      <td>0.020382</td>\n",
       "      <td>0.016810</td>\n",
       "      <td>0.029838</td>\n",
       "      <td>0.012398</td>\n",
       "      <td>0.017861</td>\n",
       "      <td>0.014499</td>\n",
       "      <td>0.016180</td>\n",
       "      <td>4759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V    RIDDLES IN THE DARK    When Bilbo...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>0.022706</td>\n",
       "      <td>0.035748</td>\n",
       "      <td>0.014905</td>\n",
       "      <td>0.020028</td>\n",
       "      <td>0.008850</td>\n",
       "      <td>0.018631</td>\n",
       "      <td>0.011295</td>\n",
       "      <td>0.014788</td>\n",
       "      <td>0.018864</td>\n",
       "      <td>0.011877</td>\n",
       "      <td>8588</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC     Anger  Anticipation   Disgust      Fear       Joy   Sadness  Surprise     Trust  word_count\n",
       "0       AN UNEXPECTED PARTY  Chapter I     AN UNEXPECTED PARTY    In a hole...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206      0.032885      0.020481  0.007885      0.023173  0.006635  0.014615  0.015288  0.011442  0.011058  0.021731       10400\n",
       "1              ROAST MUTTON  Chapter II    ROAST MUTTON    Up jumped Bilbo,...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128      0.025407      0.025407  0.015027      0.018435  0.008366  0.015182  0.013323  0.006507  0.010225  0.015957        6455\n",
       "2              A SHORT REST  Chapter III    A SHORT REST    They did not si...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846      0.039481      0.017973  0.010018      0.025928  0.008250  0.010018  0.020919  0.010018  0.013848  0.023866        3394\n",
       "3  OVER HILL AND UNDER HILL  Chapter IV    OVER HILL AND UNDER HILL    Ther...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319      0.028157      0.047489  0.019962      0.020382  0.016810  0.029838  0.012398  0.017861  0.014499  0.016180        4759\n",
       "4       RIDDLES IN THE DARK  Chapter V    RIDDLES IN THE DARK    When Bilbo...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751      0.022706      0.035748  0.014905      0.020028  0.008850  0.018631  0.011295  0.014788  0.018864  0.011877        8588"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for emotion in emotions:\n",
    "    Hobbit_sentiments_final[emotion]=Hobbit_sentiments_final[emotion]/Hobbit_sentiments_final['word_count']\n",
    "\n",
    "Hobbit_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "Hobbit_sentiments_final.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "      <th>Rating_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AN UNEXPECTED PARTY</td>\n",
       "      <td>Chapter I     AN UNEXPECTED PARTY    In a hole...</td>\n",
       "      <td>47620</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.190665</td>\n",
       "      <td>0.046048</td>\n",
       "      <td>0.858940</td>\n",
       "      <td>0.095017</td>\n",
       "      <td>0.125155</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.061535</td>\n",
       "      <td>0.391206</td>\n",
       "      <td>0.032885</td>\n",
       "      <td>0.020481</td>\n",
       "      <td>0.007885</td>\n",
       "      <td>0.023173</td>\n",
       "      <td>0.006635</td>\n",
       "      <td>0.014615</td>\n",
       "      <td>0.015288</td>\n",
       "      <td>0.011442</td>\n",
       "      <td>0.011058</td>\n",
       "      <td>0.021731</td>\n",
       "      <td>10400</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROAST MUTTON</td>\n",
       "      <td>Chapter II    ROAST MUTTON    Up jumped Bilbo,...</td>\n",
       "      <td>28863</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.229236</td>\n",
       "      <td>0.063258</td>\n",
       "      <td>0.874296</td>\n",
       "      <td>0.062453</td>\n",
       "      <td>0.004399</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.047677</td>\n",
       "      <td>0.319128</td>\n",
       "      <td>0.025407</td>\n",
       "      <td>0.025407</td>\n",
       "      <td>0.015027</td>\n",
       "      <td>0.018435</td>\n",
       "      <td>0.008366</td>\n",
       "      <td>0.015182</td>\n",
       "      <td>0.013323</td>\n",
       "      <td>0.006507</td>\n",
       "      <td>0.010225</td>\n",
       "      <td>0.015957</td>\n",
       "      <td>6455</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A SHORT REST</td>\n",
       "      <td>Chapter III    A SHORT REST    They did not si...</td>\n",
       "      <td>15671</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.275179</td>\n",
       "      <td>0.035789</td>\n",
       "      <td>0.849283</td>\n",
       "      <td>0.114947</td>\n",
       "      <td>0.136103</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.098664</td>\n",
       "      <td>0.358846</td>\n",
       "      <td>0.039481</td>\n",
       "      <td>0.017973</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.025928</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.020919</td>\n",
       "      <td>0.010018</td>\n",
       "      <td>0.013848</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>3394</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OVER HILL AND UNDER HILL</td>\n",
       "      <td>Chapter IV    OVER HILL AND UNDER HILL    Ther...</td>\n",
       "      <td>22684</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.256335</td>\n",
       "      <td>0.082478</td>\n",
       "      <td>0.848184</td>\n",
       "      <td>0.069357</td>\n",
       "      <td>-0.034112</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.389319</td>\n",
       "      <td>0.028157</td>\n",
       "      <td>0.047489</td>\n",
       "      <td>0.019962</td>\n",
       "      <td>0.020382</td>\n",
       "      <td>0.016810</td>\n",
       "      <td>0.029838</td>\n",
       "      <td>0.012398</td>\n",
       "      <td>0.017861</td>\n",
       "      <td>0.014499</td>\n",
       "      <td>0.016180</td>\n",
       "      <td>4759</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RIDDLES IN THE DARK</td>\n",
       "      <td>Chapter V    RIDDLES IN THE DARK    When Bilbo...</td>\n",
       "      <td>38059</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.181741</td>\n",
       "      <td>0.103889</td>\n",
       "      <td>0.824226</td>\n",
       "      <td>0.071892</td>\n",
       "      <td>-0.043253</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>-0.006902</td>\n",
       "      <td>0.339751</td>\n",
       "      <td>0.022706</td>\n",
       "      <td>0.035748</td>\n",
       "      <td>0.014905</td>\n",
       "      <td>0.020028</td>\n",
       "      <td>0.008850</td>\n",
       "      <td>0.018631</td>\n",
       "      <td>0.011295</td>\n",
       "      <td>0.014788</td>\n",
       "      <td>0.018864</td>\n",
       "      <td>0.011877</td>\n",
       "      <td>8588</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>OUT OF THE FRYING-PAN INTO THE FIRE</td>\n",
       "      <td>Chapter VI    OUT OF THE FRYING-PAN INTO   THE...</td>\n",
       "      <td>36814</td>\n",
       "      <td>0.200706</td>\n",
       "      <td>0.200706</td>\n",
       "      <td>0.079857</td>\n",
       "      <td>0.850042</td>\n",
       "      <td>0.070087</td>\n",
       "      <td>-0.024999</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.388771</td>\n",
       "      <td>0.031993</td>\n",
       "      <td>0.037903</td>\n",
       "      <td>0.018502</td>\n",
       "      <td>0.025826</td>\n",
       "      <td>0.019144</td>\n",
       "      <td>0.025055</td>\n",
       "      <td>0.016318</td>\n",
       "      <td>0.014519</td>\n",
       "      <td>0.018116</td>\n",
       "      <td>0.021842</td>\n",
       "      <td>7783</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>QUEER LODGINGS</td>\n",
       "      <td>Chapter VII    QUEER LODGINGS    The next morn...</td>\n",
       "      <td>48561</td>\n",
       "      <td>0.173946</td>\n",
       "      <td>0.173946</td>\n",
       "      <td>0.055134</td>\n",
       "      <td>0.858517</td>\n",
       "      <td>0.077430</td>\n",
       "      <td>0.062156</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.086392</td>\n",
       "      <td>0.337145</td>\n",
       "      <td>0.036398</td>\n",
       "      <td>0.022065</td>\n",
       "      <td>0.009712</td>\n",
       "      <td>0.025648</td>\n",
       "      <td>0.007072</td>\n",
       "      <td>0.015559</td>\n",
       "      <td>0.015276</td>\n",
       "      <td>0.010938</td>\n",
       "      <td>0.013956</td>\n",
       "      <td>0.021405</td>\n",
       "      <td>10605</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FLIES AND SPIDERS</td>\n",
       "      <td>Chapter VIII    FLIES AND SPIDERS    They walk...</td>\n",
       "      <td>55336</td>\n",
       "      <td>0.171966</td>\n",
       "      <td>0.171966</td>\n",
       "      <td>0.087934</td>\n",
       "      <td>0.843843</td>\n",
       "      <td>0.068238</td>\n",
       "      <td>-0.036351</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.019579</td>\n",
       "      <td>0.408200</td>\n",
       "      <td>0.031279</td>\n",
       "      <td>0.028219</td>\n",
       "      <td>0.016575</td>\n",
       "      <td>0.023459</td>\n",
       "      <td>0.014110</td>\n",
       "      <td>0.018785</td>\n",
       "      <td>0.016150</td>\n",
       "      <td>0.016490</td>\n",
       "      <td>0.016745</td>\n",
       "      <td>0.021164</td>\n",
       "      <td>11765</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>BARRELS OUT OF BOND</td>\n",
       "      <td>Chapter IX    BARRELS OUT OF BOND    The day a...</td>\n",
       "      <td>31430</td>\n",
       "      <td>0.214007</td>\n",
       "      <td>0.214007</td>\n",
       "      <td>0.075414</td>\n",
       "      <td>0.853004</td>\n",
       "      <td>0.071605</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.008257</td>\n",
       "      <td>0.363186</td>\n",
       "      <td>0.033364</td>\n",
       "      <td>0.024758</td>\n",
       "      <td>0.011171</td>\n",
       "      <td>0.022494</td>\n",
       "      <td>0.005888</td>\n",
       "      <td>0.016304</td>\n",
       "      <td>0.012983</td>\n",
       "      <td>0.012530</td>\n",
       "      <td>0.012077</td>\n",
       "      <td>0.022192</td>\n",
       "      <td>6624</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>A WARM WELCOME</td>\n",
       "      <td>Chapter X    A WARM WELCOME    The day grew li...</td>\n",
       "      <td>21476</td>\n",
       "      <td>0.263410</td>\n",
       "      <td>0.263410</td>\n",
       "      <td>0.057529</td>\n",
       "      <td>0.862180</td>\n",
       "      <td>0.080308</td>\n",
       "      <td>0.095816</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.090141</td>\n",
       "      <td>0.430428</td>\n",
       "      <td>0.036183</td>\n",
       "      <td>0.024197</td>\n",
       "      <td>0.008141</td>\n",
       "      <td>0.021031</td>\n",
       "      <td>0.006558</td>\n",
       "      <td>0.013569</td>\n",
       "      <td>0.010403</td>\n",
       "      <td>0.012664</td>\n",
       "      <td>0.010403</td>\n",
       "      <td>0.018544</td>\n",
       "      <td>4422</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC     Anger  Anticipation   Disgust      Fear       Joy   Sadness  Surprise     Trust  word_count  Rating_num\n",
       "0                  AN UNEXPECTED PARTY  Chapter I     AN UNEXPECTED PARTY    In a hole...        47620     0.190665          0.190665  0.046048  0.858940  0.095017  0.125155  Positive  0.061535      0.391206      0.032885      0.020481  0.007885      0.023173  0.006635  0.014615  0.015288  0.011442  0.011058  0.021731       10400           1\n",
       "1                         ROAST MUTTON  Chapter II    ROAST MUTTON    Up jumped Bilbo,...        28863     0.229236          0.229236  0.063258  0.874296  0.062453  0.004399   Neutral  0.047677      0.319128      0.025407      0.025407  0.015027      0.018435  0.008366  0.015182  0.013323  0.006507  0.010225  0.015957        6455           0\n",
       "2                         A SHORT REST  Chapter III    A SHORT REST    They did not si...        15671     0.275179          0.275179  0.035789  0.849283  0.114947  0.136103  Positive  0.098664      0.358846      0.039481      0.017973  0.010018      0.025928  0.008250  0.010018  0.020919  0.010018  0.013848  0.023866        3394           1\n",
       "3             OVER HILL AND UNDER HILL  Chapter IV    OVER HILL AND UNDER HILL    Ther...        22684     0.256335          0.256335  0.082478  0.848184  0.069357 -0.034112   Neutral  0.004561      0.389319      0.028157      0.047489  0.019962      0.020382  0.016810  0.029838  0.012398  0.017861  0.014499  0.016180        4759           0\n",
       "4                  RIDDLES IN THE DARK  Chapter V    RIDDLES IN THE DARK    When Bilbo...        38059     0.181741          0.181741  0.103889  0.824226  0.071892 -0.043253   Neutral -0.006902      0.339751      0.022706      0.035748  0.014905      0.020028  0.008850  0.018631  0.011295  0.014788  0.018864  0.011877        8588           0\n",
       "5  OUT OF THE FRYING-PAN INTO THE FIRE  Chapter VI    OUT OF THE FRYING-PAN INTO   THE...        36814     0.200706          0.200706  0.079857  0.850042  0.070087 -0.024999   Neutral  0.020121      0.388771      0.031993      0.037903  0.018502      0.025826  0.019144  0.025055  0.016318  0.014519  0.018116  0.021842        7783           0\n",
       "6                       QUEER LODGINGS  Chapter VII    QUEER LODGINGS    The next morn...        48561     0.173946          0.173946  0.055134  0.858517  0.077430  0.062156  Positive  0.086392      0.337145      0.036398      0.022065  0.009712      0.025648  0.007072  0.015559  0.015276  0.010938  0.013956  0.021405       10605           1\n",
       "7                    FLIES AND SPIDERS  Chapter VIII    FLIES AND SPIDERS    They walk...        55336     0.171966          0.171966  0.087934  0.843843  0.068238 -0.036351   Neutral  0.019579      0.408200      0.031279      0.028219  0.016575      0.023459  0.014110  0.018785  0.016150  0.016490  0.016745  0.021164       11765           0\n",
       "8                  BARRELS OUT OF BOND  Chapter IX    BARRELS OUT OF BOND    The day a...        31430     0.214007          0.214007  0.075414  0.853004  0.071605  0.019908   Neutral  0.008257      0.363186      0.033364      0.024758  0.011171      0.022494  0.005888  0.016304  0.012983  0.012530  0.012077  0.022192        6624           0\n",
       "9                       A WARM WELCOME  Chapter X    A WARM WELCOME    The day grew li...        21476     0.263410          0.263410  0.057529  0.862180  0.080308  0.095816  Positive  0.090141      0.430428      0.036183      0.024197  0.008141      0.021031  0.006558  0.013569  0.010403  0.012664  0.010403  0.018544        4422           1"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def ratings(Hobbit_sentiments_final):\n",
    "    if Hobbit_sentiments_final['Compound'] > 0.05:\n",
    "        return 1\n",
    "    elif Hobbit_sentiments_final['Compound'] < -0.05:\n",
    "        return -1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "Hobbit_sentiments_final['Rating_num'] = Hobbit_sentiments_final.apply(ratings, axis=1)\n",
    "\n",
    "Hobbit_sentiments_final.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Hobbit\\Hobbit_sentiments.csv')\n",
    "\n",
    "Hobbit_sentiments_final.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}