data-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:50.481652Z",
     "start_time": "2019-07-24T16:25:49.886691Z"
    }
   },
   "outputs": [],
   "source": [
    "# import necessary tools\n",
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:50.524990Z",
     "start_time": "2019-07-24T16:25:50.500669Z"
    }
   },
   "outputs": [],
   "source": [
    "# create file paths + dataframe\n",
    "path = './lyrics_files'\n",
    "lyrics_df = pd.DataFrame(columns=['name', 'album', 'year', 'lyrics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:51.905528Z",
     "start_time": "2019-07-24T16:25:50.531287Z"
    }
   },
   "outputs": [],
   "source": [
    "# add data to dataframe\n",
    "for filename in glob.glob(os.path.join(path, '*.json')): #only process .JSON files in folder.      \n",
    "    with open(filename, encoding='utf-8', mode='r') as json_file:\n",
    "        data = json.load(json_file)\n",
    "        data.keys()\n",
    "        songs = data.get('songs')\n",
    "        for x in songs:\n",
    "            lyrics_df = lyrics_df.append({\n",
    "                'name': x.get('title'),\n",
    "                'album': x.get('album'),\n",
    "                'year': x.get('year'),\n",
    "                'lyrics': x.get('lyrics')\n",
    "            }, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:51.982265Z",
     "start_time": "2019-07-24T16:25:51.940652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Good Ones Go (Interlude)</td>\n",
       "      <td>Take Care</td>\n",
       "      <td>2011-11-15</td>\n",
       "      <td>[Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Keep the Family Close</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>[Intro]\\nIt's a little chilly out there\\nHow y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Using Me</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yeah\\n6 shit\\n\\nBeen with so many women I can’...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Light Up</td>\n",
       "      <td>Thank Me Later</td>\n",
       "      <td>2010-06-15</td>\n",
       "      <td>[Verse 1: Drake]\\nUh, I've been up for four da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Catch No Feelings</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[Drake]\\nI told you from the jump, me and you ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>These Days</td>\n",
       "      <td>None</td>\n",
       "      <td>2016-03-29</td>\n",
       "      <td>[Verse 1]\\nI've been out walking\\nI don't do t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Overdose On Life</td>\n",
       "      <td>None</td>\n",
       "      <td>2008-04-30</td>\n",
       "      <td>And I'm a leave on one more note\\nOne more mot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Talk Up</td>\n",
       "      <td>Scorpion</td>\n",
       "      <td>2018-06-29</td>\n",
       "      <td>[Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Cannonball</td>\n",
       "      <td>Heartbreak Drake</td>\n",
       "      <td>None</td>\n",
       "      <td>[Chorus: Colin Munroe]\\nHow could I fly, but l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Thank You Note to 40</td>\n",
       "      <td>None</td>\n",
       "      <td>2016-04-28</td>\n",
       "      <td>Before we do this I just wanna say thanks man....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name             album        year  \\\n",
       "0  Good Ones Go (Interlude)         Take Care  2011-11-15   \n",
       "1     Keep the Family Close             Views  2016-04-29   \n",
       "2                  Using Me              None        None   \n",
       "3                  Light Up    Thank Me Later  2010-06-15   \n",
       "4         Catch No Feelings              None        None   \n",
       "5                These Days              None  2016-03-29   \n",
       "6          Overdose On Life              None  2008-04-30   \n",
       "7                   Talk Up          Scorpion  2018-06-29   \n",
       "8                Cannonball  Heartbreak Drake        None   \n",
       "9      Thank You Note to 40              None  2016-04-28   \n",
       "\n",
       "                                              lyrics  \n",
       "0  [Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...  \n",
       "1  [Intro]\\nIt's a little chilly out there\\nHow y...  \n",
       "2  Yeah\\n6 shit\\n\\nBeen with so many women I can’...  \n",
       "3  [Verse 1: Drake]\\nUh, I've been up for four da...  \n",
       "4  [Drake]\\nI told you from the jump, me and you ...  \n",
       "5  [Verse 1]\\nI've been out walking\\nI don't do t...  \n",
       "6  And I'm a leave on one more note\\nOne more mot...  \n",
       "7  [Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...  \n",
       "8  [Chorus: Colin Munroe]\\nHow could I fly, but l...  \n",
       "9  Before we do this I just wanna say thanks man....  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.005255Z",
     "start_time": "2019-07-24T16:25:51.988546Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 392 entries, 0 to 391\n",
      "Data columns (total 4 columns):\n",
      "name      392 non-null object\n",
      "album     241 non-null object\n",
      "year      280 non-null object\n",
      "lyrics    392 non-null object\n",
      "dtypes: object(4)\n",
      "memory usage: 12.3+ KB\n"
     ]
    }
   ],
   "source": [
    "lyrics_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will only use officially released songs (songs that are in an album) for this project; therefore, we remove the rows that have null values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.024717Z",
     "start_time": "2019-07-24T16:25:52.015799Z"
    }
   },
   "outputs": [],
   "source": [
    "# remove all rows that have null values\n",
    "lyrics_df = lyrics_df.dropna(how='any', axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.049401Z",
     "start_time": "2019-07-24T16:25:52.036174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 221 entries, 0 to 390\n",
      "Data columns (total 4 columns):\n",
      "name      221 non-null object\n",
      "album     221 non-null object\n",
      "year      221 non-null object\n",
      "lyrics    221 non-null object\n",
      "dtypes: object(4)\n",
      "memory usage: 8.6+ KB\n"
     ]
    }
   ],
   "source": [
    "lyrics_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.076585Z",
     "start_time": "2019-07-24T16:25:52.055756Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Good Ones Go (Interlude)</td>\n",
       "      <td>Take Care</td>\n",
       "      <td>2011-11-15</td>\n",
       "      <td>[Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Keep the Family Close</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>[Intro]\\nIt's a little chilly out there\\nHow y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Light Up</td>\n",
       "      <td>Thank Me Later</td>\n",
       "      <td>2010-06-15</td>\n",
       "      <td>[Verse 1: Drake]\\nUh, I've been up for four da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Talk Up</td>\n",
       "      <td>Scorpion</td>\n",
       "      <td>2018-06-29</td>\n",
       "      <td>[Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Juice</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009-12-11</td>\n",
       "      <td>[Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Come Thru</td>\n",
       "      <td>Nothing Was the Same</td>\n",
       "      <td>2013-09-24</td>\n",
       "      <td>[Intro]\\nYeah\\nWe had the type of nights where...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Summer Games</td>\n",
       "      <td>Scorpion</td>\n",
       "      <td>2018-06-29</td>\n",
       "      <td>[Chorus]\\nI think you're changin' your mind\\nS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>[Intro]\\nYeah, for the dogs dem, you know\\nMan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Glow</td>\n",
       "      <td>More Life</td>\n",
       "      <td>2017-03-18</td>\n",
       "      <td>[Chorus: Kanye West]\\nWatch out for me, I'm bo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Hotline Bling</td>\n",
       "      <td>Views</td>\n",
       "      <td>2015-07-25</td>\n",
       "      <td>[Intro]\\nYou used to call me on my\\nYou used t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        name                 album        year  \\\n",
       "0   Good Ones Go (Interlude)             Take Care  2011-11-15   \n",
       "1      Keep the Family Close                 Views  2016-04-29   \n",
       "3                   Light Up        Thank Me Later  2010-06-15   \n",
       "7                    Talk Up              Scorpion  2018-06-29   \n",
       "10                     Juice       Born Successful  2009-12-11   \n",
       "13                 Come Thru  Nothing Was the Same  2013-09-24   \n",
       "14              Summer Games              Scorpion  2018-06-29   \n",
       "15                         9                 Views  2016-04-29   \n",
       "17                      Glow             More Life  2017-03-18   \n",
       "19             Hotline Bling                 Views  2015-07-25   \n",
       "\n",
       "                                               lyrics  \n",
       "0   [Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...  \n",
       "1   [Intro]\\nIt's a little chilly out there\\nHow y...  \n",
       "3   [Verse 1: Drake]\\nUh, I've been up for four da...  \n",
       "7   [Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...  \n",
       "10  [Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...  \n",
       "13  [Intro]\\nYeah\\nWe had the type of nights where...  \n",
       "14  [Chorus]\\nI think you're changin' your mind\\nS...  \n",
       "15  [Intro]\\nYeah, for the dogs dem, you know\\nMan...  \n",
       "17  [Chorus: Kanye West]\\nWatch out for me, I'm bo...  \n",
       "19  [Intro]\\nYou used to call me on my\\nYou used t...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's make the 'year' column year only:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.155623Z",
     "start_time": "2019-07-24T16:25:52.082109Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Good Ones Go (Interlude)</td>\n",
       "      <td>Take Care</td>\n",
       "      <td>2011</td>\n",
       "      <td>[Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Keep the Family Close</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>[Intro]\\nIt's a little chilly out there\\nHow y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Light Up</td>\n",
       "      <td>Thank Me Later</td>\n",
       "      <td>2010</td>\n",
       "      <td>[Verse 1: Drake]\\nUh, I've been up for four da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Talk Up</td>\n",
       "      <td>Scorpion</td>\n",
       "      <td>2018</td>\n",
       "      <td>[Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Juice</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Come Thru</td>\n",
       "      <td>Nothing Was the Same</td>\n",
       "      <td>2013</td>\n",
       "      <td>[Intro]\\nYeah\\nWe had the type of nights where...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Summer Games</td>\n",
       "      <td>Scorpion</td>\n",
       "      <td>2018</td>\n",
       "      <td>[Chorus]\\nI think you're changin' your mind\\nS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>[Intro]\\nYeah, for the dogs dem, you know\\nMan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Glow</td>\n",
       "      <td>More Life</td>\n",
       "      <td>2017</td>\n",
       "      <td>[Chorus: Kanye West]\\nWatch out for me, I'm bo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Hotline Bling</td>\n",
       "      <td>Views</td>\n",
       "      <td>2015</td>\n",
       "      <td>[Intro]\\nYou used to call me on my\\nYou used t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        name                 album  year  \\\n",
       "0   Good Ones Go (Interlude)             Take Care  2011   \n",
       "1      Keep the Family Close                 Views  2016   \n",
       "3                   Light Up        Thank Me Later  2010   \n",
       "7                    Talk Up              Scorpion  2018   \n",
       "10                     Juice       Born Successful  2009   \n",
       "13                 Come Thru  Nothing Was the Same  2013   \n",
       "14              Summer Games              Scorpion  2018   \n",
       "15                         9                 Views  2016   \n",
       "17                      Glow             More Life  2017   \n",
       "19             Hotline Bling                 Views  2015   \n",
       "\n",
       "                                               lyrics  \n",
       "0   [Intro]\\nOh yeah\\nOh yeah, oh yeah, oh\\nOh yea...  \n",
       "1   [Intro]\\nIt's a little chilly out there\\nHow y...  \n",
       "3   [Verse 1: Drake]\\nUh, I've been up for four da...  \n",
       "7   [Intro: Drake]\\nDJ Paul\\nTWhy\\nAyy, ayy\\nLot o...  \n",
       "10  [Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...  \n",
       "13  [Intro]\\nYeah\\nWe had the type of nights where...  \n",
       "14  [Chorus]\\nI think you're changin' your mind\\nS...  \n",
       "15  [Intro]\\nYeah, for the dogs dem, you know\\nMan...  \n",
       "17  [Chorus: Kanye West]\\nWatch out for me, I'm bo...  \n",
       "19  [Intro]\\nYou used to call me on my\\nYou used t...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df['year'] = pd.to_datetime(lyrics_df['year']).dt.strftime('%Y')\n",
    "lyrics_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Re-sort the data alphabetically regarding the album names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.208410Z",
     "start_time": "2019-07-24T16:25:52.169512Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>Right to Left</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro]\\n[?]\\n\\n[Chorus: Drake]\\nBlue green je...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>Forever (Born Successful)</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Chorus: Drake]\\nIt may not mean nothing to y'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>The Winner</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Verse 1]\\nI’m performing tonight you know tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>I Do This</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Verse 1: Drake]\\nUh, shit's all good\\nThe dea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>Fallen</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro: Drake]\\nYeah, it's Drake, KC\\n\\n[Verse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>Do It Now</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro]\\nUh, yeah, alright\\nUh, well alright\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>The Search</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>(Saukrates intro)\\nThey say we killin' 'em all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Juice</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>Man of the Year</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>[Verse 1: Drake]\\nDamn, I done walked in here\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>Give Ya</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>[Verse 1: Drake]\\nCheck, look\\nAnd I ain't try...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          name            album  year  \\\n",
       "155              Right to Left  Born Successful  2009   \n",
       "200  Forever (Born Successful)  Born Successful  2009   \n",
       "189                 The Winner  Born Successful  2009   \n",
       "181                  I Do This  Born Successful  2009   \n",
       "120                     Fallen  Born Successful  2009   \n",
       "328                  Do It Now  Born Successful  2009   \n",
       "286                 The Search  Born Successful  2009   \n",
       "10                       Juice  Born Successful  2009   \n",
       "203            Man of the Year  Comeback Season  2007   \n",
       "168                    Give Ya  Comeback Season  2007   \n",
       "\n",
       "                                                lyrics  \n",
       "155  [Intro]\\n[?]\\n\\n[Chorus: Drake]\\nBlue green je...  \n",
       "200  [Chorus: Drake]\\nIt may not mean nothing to y'...  \n",
       "189  [Verse 1]\\nI’m performing tonight you know tha...  \n",
       "181  [Verse 1: Drake]\\nUh, shit's all good\\nThe dea...  \n",
       "120  [Intro: Drake]\\nYeah, it's Drake, KC\\n\\n[Verse...  \n",
       "328  [Intro]\\nUh, yeah, alright\\nUh, well alright\\n...  \n",
       "286  (Saukrates intro)\\nThey say we killin' 'em all...  \n",
       "10   [Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...  \n",
       "203  [Verse 1: Drake]\\nDamn, I done walked in here\\...  \n",
       "168  [Verse 1: Drake]\\nCheck, look\\nAnd I ain't try...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df = lyrics_df.sort_values('album')\n",
    "lyrics_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We reset the index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.239727Z",
     "start_time": "2019-07-24T16:25:52.217438Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Right to Left</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro]\\n[?]\\n\\n[Chorus: Drake]\\nBlue green je...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Forever (Born Successful)</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Chorus: Drake]\\nIt may not mean nothing to y'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Winner</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Verse 1]\\nI’m performing tonight you know tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I Do This</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Verse 1: Drake]\\nUh, shit's all good\\nThe dea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fallen</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro: Drake]\\nYeah, it's Drake, KC\\n\\n[Verse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Do It Now</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Intro]\\nUh, yeah, alright\\nUh, well alright\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The Search</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>(Saukrates intro)\\nThey say we killin' 'em all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Juice</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>[Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Man of the Year</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>[Verse 1: Drake]\\nDamn, I done walked in here\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Give Ya</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>[Verse 1: Drake]\\nCheck, look\\nAnd I ain't try...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        name            album  year  \\\n",
       "0              Right to Left  Born Successful  2009   \n",
       "1  Forever (Born Successful)  Born Successful  2009   \n",
       "2                 The Winner  Born Successful  2009   \n",
       "3                  I Do This  Born Successful  2009   \n",
       "4                     Fallen  Born Successful  2009   \n",
       "5                  Do It Now  Born Successful  2009   \n",
       "6                 The Search  Born Successful  2009   \n",
       "7                      Juice  Born Successful  2009   \n",
       "8            Man of the Year  Comeback Season  2007   \n",
       "9                    Give Ya  Comeback Season  2007   \n",
       "\n",
       "                                              lyrics  \n",
       "0  [Intro]\\n[?]\\n\\n[Chorus: Drake]\\nBlue green je...  \n",
       "1  [Chorus: Drake]\\nIt may not mean nothing to y'...  \n",
       "2  [Verse 1]\\nI’m performing tonight you know tha...  \n",
       "3  [Verse 1: Drake]\\nUh, shit's all good\\nThe dea...  \n",
       "4  [Intro: Drake]\\nYeah, it's Drake, KC\\n\\n[Verse...  \n",
       "5  [Intro]\\nUh, yeah, alright\\nUh, well alright\\n...  \n",
       "6  (Saukrates intro)\\nThey say we killin' 'em all...  \n",
       "7  [Produced by Boi-1da]\\n\\n[Intro]\\nBoi-1da... D...  \n",
       "8  [Verse 1: Drake]\\nDamn, I done walked in here\\...  \n",
       "9  [Verse 1: Drake]\\nCheck, look\\nAnd I ain't try...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df = lyrics_df.reset_index(drop=True)\n",
    "lyrics_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing additional text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that we still have some works to do with the lyrics, such as text like \"Verse 1: Drake\", \"Intro\", \"Chorus\", or even brackets and punctuations. Let's use regular expressions to clean the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:52.624770Z",
     "start_time": "2019-07-24T16:25:52.589224Z"
    }
   },
   "outputs": [],
   "source": [
    "def clean_lyrics(lyrics):\n",
    "    if lyrics is None:\n",
    "        return lyrics\n",
    "    \n",
    "    # combine lists of tokens into single string\n",
    "    #lyrics = ' '.join(lyrics)\n",
    "            \n",
    "    # remove apostrophes\n",
    "    lyrics = lyrics.replace('\\'', '')\n",
    "            \n",
    "    # remove song structure tags or instructions in brackets\n",
    "    lyrics = re.sub(r'[\\*\\[|\\(|\\{].*\\n*.*[\\]\\)\\}\\*]' , ' ', lyrics)\n",
    "   \n",
    "    # remove variations of Verse 1, VERSE 2, etc...\n",
    "    for verse in ['verse', 'VERSE', 'Verse']:\n",
    "        lyrics = re.sub(verse+' \\d*', '', lyrics)\n",
    "    \n",
    "    # some structure markers formatted as allcaps without brackets\n",
    "    for word in ['OUTRO', 'INSTRUMENTAL', 'PRE', 'HOOK',\n",
    "                 'PRODUCED', 'REFRAIN', 'POST', 'REPEAT', '2x', '3x', '4x',\n",
    "                 'CHORUS', 'INTRO', 'INTERLUDE']:\n",
    "        lyrics = lyrics.replace(word, '')\n",
    "        \n",
    "    # remove varations of Chorus\n",
    "    lyrics = re.sub(r'\\n*Chorus:*.*' , ' ', lyrics)\n",
    "    lyrics = re.sub(r'^Chorus:*.*' , ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nRepeat [C|c]horus:*.*' , ' ', lyrics)\n",
    "    \n",
    "    # remove variations of Intro\n",
    "    lyrics = re.sub(r'Intro[\\s|\\n|:].*', ' ', lyrics)\n",
    "    \n",
    "    # remove variations of Instrumental\n",
    "    lyrics = re.sub(r'-+.*[i|I]nstrumental.*-+', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nBrief instrumental.*\\n', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nInstrumental', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nInstrumental break', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nInstrumental--', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\n~Instrumental~', ' ', lyrics)\n",
    "    \n",
    "    # remove variations of Bridge\n",
    "    lyrics = re.sub(r'\\n\\[*Bridge:\\[*', ' ', lyrics)\n",
    "    \n",
    "    # remove variations of Hook\n",
    "    lyrics = re.sub(r'Hook:.*', ' ', lyrics)\n",
    "    \n",
    "    # remove varations of Repeat\n",
    "    lyrics = re.sub(r'Repeat\\s.*', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nRepeat$', ' ', lyrics)\n",
    "    \n",
    "    # remove credits\n",
    "    lyrics = re.sub(r'.*[P|p]roduced [B|b]y.*', ' ', lyrics)\n",
    "    lyrics = re.sub(r'.*[W|w]ritten [B|b]y.*', ' ', lyrics)\n",
    "    \n",
    "    # remove strays and typos\n",
    "    lyrics = re.sub(r'\\[Outro\\[', ' ', lyrics)\n",
    "    lyrics = re.sub(r'Sax & background & instrumental\\)', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nSource: ', ' ', lyrics)\n",
    "    lyrics = re.sub(r'Shotgun 2: 58 Trk 1 \\n  \\nJr. Walker & The All Stars '\\\n",
    "                    +'\\nAnd/or The Funk Brothers - instrumental \\nPop Chart '\\\n",
    "                    +'#4 Feb 13, 1965 \\nSoul Label - 35008   \\n ', ' ', lyrics)\n",
    "    lyrics = re.sub(r'- musical interlude -', ' ', lyrics)\n",
    "    lyrics = re.sub(r'\\nRefrain:', ' ', lyrics)\n",
    "            \n",
    "    # replace all punctuations with spaces\n",
    "    lyrics = re.sub(r'[^\\w\\s]', ' ', lyrics)\n",
    "            \n",
    "    # replace consecutive whitespaces with single space\n",
    "    lyrics = re.sub(r'\\s+', ' ', lyrics)\n",
    "    \n",
    "    # convert all tokens to lowercase\n",
    "    lyrics = lyrics.lower()\n",
    "\n",
    "    return lyrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:53.585507Z",
     "start_time": "2019-07-24T16:25:52.639797Z"
    }
   },
   "outputs": [],
   "source": [
    "lyrics_df['lyrics'] = lyrics_df['lyrics'].map(clean_lyrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:25:53.644207Z",
     "start_time": "2019-07-24T16:25:53.595201Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Right to Left</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>blue green jewels with the supreme fuel and l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Forever (Born Successful)</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>it may not mean nothing to yall but understan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Winner</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>i m performing tonight you know that shit gon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I Do This</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>uh shits all good the deal got signed and my ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fallen</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>yeah its drake kc we was just walking just sm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Do It Now</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>uh yeah alright uh well alright yeah well alr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>The Search</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>they say we killin em all all all all hip hop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Juice</td>\n",
       "      <td>Born Successful</td>\n",
       "      <td>2009</td>\n",
       "      <td>boi 1da drizzy yall dont really like me i can...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Man of the Year</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>damn i done walked in here looking like the m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Give Ya</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>check look and i aint tryna get to know nobod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>i got ice all over my windshield i can tell i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Don’t You Have a Man</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>why are you trying to be my girl i cant belie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Intro</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>you see the difference between me and you is t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>The Last Hope</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>i gotta make it out or move quickly cant allo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Think Good Thoughts</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>yeah 9th wonder dont judge me man they tend t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Must Hate Money</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>ive got it made thats what im talking about g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Where to Now</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>when you gotta be like you know ive come this...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Faded</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>please d d d d damn what did i consume someth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Closer</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>yeah i remember me and me and d used to talk ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Replacement Girl</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>ladies make some noise if you wanna be my rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Easy to Please</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>yeah look me and sos we from two different wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Underdog</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>you already know what it is aye man look i ai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Asthma Team</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>i swear to you lord same dudes used to laugh a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>The Presentation</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>shake up the world that is what im about to d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Share</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>take what you want take what you want take wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Bitch Is Crazy</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>aww man ho yes hahaha yeah can yall hear me o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Going in for Life</td>\n",
       "      <td>Comeback Season</td>\n",
       "      <td>2007</td>\n",
       "      <td>i dont think they really ready for my wardrobe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Take Your Girl</td>\n",
       "      <td>Heartbreak Drake</td>\n",
       "      <td>2009</td>\n",
       "      <td>now shawty got a man but i got a plan look ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Closer To My Dreams</td>\n",
       "      <td>Heartbreak Drake</td>\n",
       "      <td>2009</td>\n",
       "      <td>yeah i remember me and me and d used to talk ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Stunt Hard</td>\n",
       "      <td>Heartbreak Drake</td>\n",
       "      <td>2009</td>\n",
       "      <td>i did it all on my own turned a chair to a th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>Fancy</td>\n",
       "      <td>Thank Me Later</td>\n",
       "      <td>2010</td>\n",
       "      <td>go go head go go go go go go head go go go go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>The Resistance</td>\n",
       "      <td>Thank Me Later</td>\n",
       "      <td>2010</td>\n",
       "      <td>yesterday when we were getting high you were ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>Omertà</td>\n",
       "      <td>The Best In The World Pack</td>\n",
       "      <td>2019</td>\n",
       "      <td>look at my history im tryna see whats differe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>Money in the Grave</td>\n",
       "      <td>The Best In The World Pack</td>\n",
       "      <td>2019</td>\n",
       "      <td>yeah okay lil cc on the beat mmm yeah yeah i ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>Can I?</td>\n",
       "      <td>Unreleased / Miscellaneous</td>\n",
       "      <td>2015</td>\n",
       "      <td>can i baby can i baby can i baby can i baby c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>Faithful</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>check one two one two bitch you expensive you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>Keep the Family Close</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>its a little chilly out there how you can sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>Weston Road Flows</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>one of dem ones right weston road flows i did...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>9</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>yeah for the dogs dem you know man when shit ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>Hotline Bling</td>\n",
       "      <td>Views</td>\n",
       "      <td>2015</td>\n",
       "      <td>you used to call me on my you used to you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>U With Me?</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>what these bitches want from a nigga on some ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>Fire &amp; Desire</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>dedicate i dedicate the world yeah oh i dedic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>Child’s Play</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>breaking news my niggas if your girlfriend ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>Still Here</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>doin well dog yeah me and all my yeah me yeah...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>Hype</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>man dont live of a di hype huh real ting ay y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>Feel No Ways</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>i should be downtown whipping on the way to y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>Pop Style</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>this sound like some forty three oh one shit ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Grammys</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>yeah yeah yeah yeah jheeze yeah right look lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>Redemption</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>yeah i get it i get it yeah why would i say a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>Too Good</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>oh yeah yeah yeah oh yeah yeah yeah yeah look...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>Controlla</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>right my yiy just changed you just buzzed the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>Views [Trailer]</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>the 6 raptors diamond key new ride old ride ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>Summers Over Interlude</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>ooh baby yeah days in the sun and nights in t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>Views</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>question is will i ever leave you the answer ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>With You</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>its about us right now girl where you going i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>One Dance</td>\n",
       "      <td>Views</td>\n",
       "      <td>2016</td>\n",
       "      <td>baby i like your style grips on your waist fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Trophies</td>\n",
       "      <td>Young Money: Rise Of An Empire</td>\n",
       "      <td>2014</td>\n",
       "      <td>had hit records on my demo did yall boys not ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Loving You No More</td>\n",
       "      <td>Young Sweet Jones 2</td>\n",
       "      <td>2010</td>\n",
       "      <td>aint nothin worse than the hurt we receive fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>What If I Kissed You</td>\n",
       "      <td>Young Sweet Jones 2</td>\n",
       "      <td>2010</td>\n",
       "      <td>what if i kissed you right now how would we ex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>I’m Ready For You</td>\n",
       "      <td>Young Sweet Jones 2</td>\n",
       "      <td>2010</td>\n",
       "      <td>r i p to the girl you used to see her days ar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>221 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          name                           album  year  \\\n",
       "0                Right to Left                 Born Successful  2009   \n",
       "1    Forever (Born Successful)                 Born Successful  2009   \n",
       "2                   The Winner                 Born Successful  2009   \n",
       "3                    I Do This                 Born Successful  2009   \n",
       "4                       Fallen                 Born Successful  2009   \n",
       "5                    Do It Now                 Born Successful  2009   \n",
       "6                   The Search                 Born Successful  2009   \n",
       "7                        Juice                 Born Successful  2009   \n",
       "8              Man of the Year                 Comeback Season  2007   \n",
       "9                      Give Ya                 Comeback Season  2007   \n",
       "10             Comeback Season                 Comeback Season  2007   \n",
       "11        Don’t You Have a Man                 Comeback Season  2007   \n",
       "12                       Intro                 Comeback Season  2007   \n",
       "13               The Last Hope                 Comeback Season  2007   \n",
       "14         Think Good Thoughts                 Comeback Season  2007   \n",
       "15             Must Hate Money                 Comeback Season  2007   \n",
       "16                Where to Now                 Comeback Season  2007   \n",
       "17                       Faded                 Comeback Season  2007   \n",
       "18                      Closer                 Comeback Season  2007   \n",
       "19            Replacement Girl                 Comeback Season  2007   \n",
       "20              Easy to Please                 Comeback Season  2007   \n",
       "21                    Underdog                 Comeback Season  2007   \n",
       "22                 Asthma Team                 Comeback Season  2007   \n",
       "23            The Presentation                 Comeback Season  2007   \n",
       "24                       Share                 Comeback Season  2007   \n",
       "25              Bitch Is Crazy                 Comeback Season  2007   \n",
       "26           Going in for Life                 Comeback Season  2007   \n",
       "27              Take Your Girl                Heartbreak Drake  2009   \n",
       "28         Closer To My Dreams                Heartbreak Drake  2009   \n",
       "29                  Stunt Hard                Heartbreak Drake  2009   \n",
       "..                         ...                             ...   ...   \n",
       "191                      Fancy                  Thank Me Later  2010   \n",
       "192             The Resistance                  Thank Me Later  2010   \n",
       "193                     Omertà      The Best In The World Pack  2019   \n",
       "194         Money in the Grave      The Best In The World Pack  2019   \n",
       "195                     Can I?      Unreleased / Miscellaneous  2015   \n",
       "196                   Faithful                           Views  2016   \n",
       "197      Keep the Family Close                           Views  2016   \n",
       "198          Weston Road Flows                           Views  2016   \n",
       "199                          9                           Views  2016   \n",
       "200              Hotline Bling                           Views  2015   \n",
       "201                 U With Me?                           Views  2016   \n",
       "202              Fire & Desire                           Views  2016   \n",
       "203               Child’s Play                           Views  2016   \n",
       "204                 Still Here                           Views  2016   \n",
       "205                       Hype                           Views  2016   \n",
       "206               Feel No Ways                           Views  2016   \n",
       "207                  Pop Style                           Views  2016   \n",
       "208                    Grammys                           Views  2016   \n",
       "209                 Redemption                           Views  2016   \n",
       "210                   Too Good                           Views  2016   \n",
       "211                  Controlla                           Views  2016   \n",
       "212            Views [Trailer]                           Views  2016   \n",
       "213     Summers Over Interlude                           Views  2016   \n",
       "214                      Views                           Views  2016   \n",
       "215                   With You                           Views  2016   \n",
       "216                  One Dance                           Views  2016   \n",
       "217                   Trophies  Young Money: Rise Of An Empire  2014   \n",
       "218         Loving You No More             Young Sweet Jones 2  2010   \n",
       "219       What If I Kissed You             Young Sweet Jones 2  2010   \n",
       "220          I’m Ready For You             Young Sweet Jones 2  2010   \n",
       "\n",
       "                                                lyrics  \n",
       "0     blue green jewels with the supreme fuel and l...  \n",
       "1     it may not mean nothing to yall but understan...  \n",
       "2     i m performing tonight you know that shit gon...  \n",
       "3     uh shits all good the deal got signed and my ...  \n",
       "4     yeah its drake kc we was just walking just sm...  \n",
       "5     uh yeah alright uh well alright yeah well alr...  \n",
       "6     they say we killin em all all all all hip hop...  \n",
       "7     boi 1da drizzy yall dont really like me i can...  \n",
       "8     damn i done walked in here looking like the m...  \n",
       "9     check look and i aint tryna get to know nobod...  \n",
       "10    i got ice all over my windshield i can tell i...  \n",
       "11    why are you trying to be my girl i cant belie...  \n",
       "12   you see the difference between me and you is t...  \n",
       "13    i gotta make it out or move quickly cant allo...  \n",
       "14    yeah 9th wonder dont judge me man they tend t...  \n",
       "15    ive got it made thats what im talking about g...  \n",
       "16    when you gotta be like you know ive come this...  \n",
       "17    please d d d d damn what did i consume someth...  \n",
       "18    yeah i remember me and me and d used to talk ...  \n",
       "19    ladies make some noise if you wanna be my rep...  \n",
       "20    yeah look me and sos we from two different wa...  \n",
       "21    you already know what it is aye man look i ai...  \n",
       "22   i swear to you lord same dudes used to laugh a...  \n",
       "23    shake up the world that is what im about to d...  \n",
       "24    take what you want take what you want take wh...  \n",
       "25    aww man ho yes hahaha yeah can yall hear me o...  \n",
       "26   i dont think they really ready for my wardrobe...  \n",
       "27    now shawty got a man but i got a plan look ho...  \n",
       "28    yeah i remember me and me and d used to talk ...  \n",
       "29    i did it all on my own turned a chair to a th...  \n",
       "..                                                 ...  \n",
       "191   go go head go go go go go go head go go go go...  \n",
       "192   yesterday when we were getting high you were ...  \n",
       "193   look at my history im tryna see whats differe...  \n",
       "194   yeah okay lil cc on the beat mmm yeah yeah i ...  \n",
       "195   can i baby can i baby can i baby can i baby c...  \n",
       "196   check one two one two bitch you expensive you...  \n",
       "197   its a little chilly out there how you can sta...  \n",
       "198   one of dem ones right weston road flows i did...  \n",
       "199   yeah for the dogs dem you know man when shit ...  \n",
       "200   you used to call me on my you used to you use...  \n",
       "201   what these bitches want from a nigga on some ...  \n",
       "202   dedicate i dedicate the world yeah oh i dedic...  \n",
       "203   breaking news my niggas if your girlfriend ha...  \n",
       "204   doin well dog yeah me and all my yeah me yeah...  \n",
       "205   man dont live of a di hype huh real ting ay y...  \n",
       "206   i should be downtown whipping on the way to y...  \n",
       "207   this sound like some forty three oh one shit ...  \n",
       "208   yeah yeah yeah yeah jheeze yeah right look lo...  \n",
       "209   yeah i get it i get it yeah why would i say a...  \n",
       "210   oh yeah yeah yeah oh yeah yeah yeah yeah look...  \n",
       "211   right my yiy just changed you just buzzed the...  \n",
       "212  the 6 raptors diamond key new ride old ride ba...  \n",
       "213   ooh baby yeah days in the sun and nights in t...  \n",
       "214   question is will i ever leave you the answer ...  \n",
       "215   its about us right now girl where you going i...  \n",
       "216   baby i like your style grips on your waist fr...  \n",
       "217   had hit records on my demo did yall boys not ...  \n",
       "218   aint nothin worse than the hurt we receive fr...  \n",
       "219  what if i kissed you right now how would we ex...  \n",
       "220   r i p to the girl you used to see her days ar...  \n",
       "\n",
       "[221 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks good! Now we save the data as a csv file and move on to the next steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-24T16:37:58.788737Z",
     "start_time": "2019-07-24T16:37:58.727490Z"
    }
   },
   "outputs": [],
   "source": [
    "# add data to csv file\n",
    "lyrics_path = \"./lyrics.csv\"\n",
    "lyrics_df.to_csv(lyrics_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}