Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dta = pd.read_csv(\"master-process-data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = dta[['genre','lyrics']]\n",
    "dta.dropna(subset=['genre', 'lyrics'], inplace=True)\n",
    "\n",
    "#replace new line with space\n",
    "dta = dta.replace({'\\n': ' '}, regex=True)\n",
    "\n",
    "#only have strings as lyrics\n",
    "dta = dta[dta['lyrics'].apply(type) == str]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rock             109221\n",
      "rap               53557\n",
      "Pop               40466\n",
      "Hip-Hop           24849\n",
      "Not Available     23934\n",
      "Metal             23759\n",
      "Country           14387\n",
      "Jazz               7971\n",
      "Electronic         7966\n",
      "Other              5189\n",
      "R&B                3401\n",
      "Indie              3149\n",
      "Folk               2243\n",
      "Name: genre, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#Songs per genre\n",
    "print(dta['genre'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "genre\n",
      "Country          185.787099\n",
      "Electronic       193.070550\n",
      "Folk             180.422648\n",
      "Hip-Hop          489.371645\n",
      "Indie            196.710384\n",
      "Jazz             171.952954\n",
      "Metal            168.438108\n",
      "Not Available    209.335798\n",
      "Other            223.667566\n",
      "Pop              245.412050\n",
      "R&B              224.786827\n",
      "Rock             190.341143\n",
      "rap              310.597662\n",
      "Name: word_num, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Average length of songs in each genre\n",
    "dta['word_num'] = dta['lyrics'].str.split().str.len()\n",
    "avg_len = dta.groupby('genre')['word_num'].mean()\n",
    "dta = dta.drop(columns = ['word_num'])\n",
    "print(avg_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Metal Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 210628),\n",
       " ('i', 94608),\n",
       " ('to', 92835),\n",
       " ('of', 86348),\n",
       " ('you', 82233),\n",
       " ('and', 77236),\n",
       " ('a', 66455),\n",
       " ('in', 60893),\n",
       " ('my', 55688),\n",
       " ('your', 43802),\n",
       " ('me', 42062),\n",
       " ('is', 39997),\n",
       " ('for', 32104),\n",
       " ('it', 29504),\n",
       " ('all', 27296),\n",
       " ('we', 25380),\n",
       " ('on', 25226),\n",
       " ('this', 24877),\n",
       " ('that', 24170),\n",
       " ('common', 23942),\n",
       " ('metal:', 23758),\n",
       " ('will', 22767),\n",
       " ('with', 22007),\n",
       " ('be', 21276),\n",
       " ('no', 21137),\n",
       " (\"i'm\", 18065),\n",
       " ('are', 18004),\n",
       " ('so', 17830),\n",
       " ('from', 17778),\n",
       " ('but', 15340),\n",
       " ('now', 15040),\n",
       " ('what', 14323),\n",
       " ('as', 13714),\n",
       " (\"it's\", 13663),\n",
       " ('life', 13445),\n",
       " ('one', 12876),\n",
       " ('time', 12860),\n",
       " ('by', 12593),\n",
       " (\"don't\", 12582),\n",
       " ('they', 12462),\n",
       " ('not', 12287),\n",
       " ('see', 12222),\n",
       " ('our', 12174),\n",
       " ('never', 12069),\n",
       " ('like', 12005),\n",
       " ('when', 11622),\n",
       " ('have', 11499),\n",
       " ('can', 11465),\n",
       " ('out', 11201),\n",
       " ('just', 10989),\n",
       " ('know', 10474),\n",
       " ('up', 10159),\n",
       " ('was', 9793),\n",
       " ('down', 9731),\n",
       " ('at', 9030),\n",
       " ('world', 8335),\n",
       " ('do', 8230),\n",
       " ('am', 8211),\n",
       " ('their', 8163),\n",
       " ('away', 8152),\n",
       " ('eyes', 8103),\n",
       " (\"can't\", 8059),\n",
       " ('through', 8052),\n",
       " (\"you're\", 8048),\n",
       " ('take', 7892),\n",
       " ('die', 7582),\n",
       " ('into', 7531),\n",
       " ('come', 7519),\n",
       " ('way', 7474),\n",
       " ('there', 7333),\n",
       " ('love', 7325),\n",
       " ('if', 7169),\n",
       " ('feel', 7038),\n",
       " ('back', 6805),\n",
       " ('his', 6775),\n",
       " ('us', 6719),\n",
       " ('where', 6672),\n",
       " ('blood', 6550),\n",
       " ('let', 6494),\n",
       " ('more', 6404),\n",
       " ('he', 6366),\n",
       " (\"i'll\", 6324),\n",
       " ('an', 6296),\n",
       " ('death', 6236),\n",
       " ('her', 6213),\n",
       " ('get', 6175),\n",
       " ('who', 6159),\n",
       " ('go', 6122),\n",
       " ('want', 6000),\n",
       " ('night', 5992),\n",
       " ('only', 5925),\n",
       " ('here', 5846),\n",
       " ('mind', 5798),\n",
       " ('pain', 5762),\n",
       " ('has', 5729),\n",
       " ('day', 5705),\n",
       " ('again', 5674),\n",
       " ('or', 5572),\n",
       " ('make', 5536),\n",
       " (\"i've\", 5481)]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common metal word\n",
    "print(\"Most Common Metal Words: \")\n",
    "Counter(\"Most common metal: \\n \".join(dta.loc[dta['genre']=='Metal','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Rap Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 581723),\n",
       " ('i', 581385),\n",
       " ('you', 543237),\n",
       " ('and', 341599),\n",
       " ('to', 336701),\n",
       " ('a', 327513),\n",
       " ('me', 269735),\n",
       " ('my', 248757),\n",
       " ('it', 225884),\n",
       " ('in', 202607),\n",
       " ('on', 167148),\n",
       " ('that', 166719),\n",
       " (\"i'm\", 158552),\n",
       " ('of', 140781),\n",
       " ('your', 139002),\n",
       " ('all', 114892),\n",
       " ('like', 113253),\n",
       " ('be', 111492),\n",
       " ('for', 110151),\n",
       " ('we', 106683),\n",
       " ('love', 106463),\n",
       " (\"don't\", 106261),\n",
       " ('is', 105542),\n",
       " ('with', 102690),\n",
       " ('know', 101093),\n",
       " ('up', 99786),\n",
       " ('so', 93774),\n",
       " ('but', 92127),\n",
       " ('yeah', 91588),\n",
       " ('got', 89995),\n",
       " ('no', 89927),\n",
       " ('just', 87898),\n",
       " ('this', 82071),\n",
       " ('get', 80879),\n",
       " (\"it's\", 78804),\n",
       " ('oh', 75817),\n",
       " ('when', 75768),\n",
       " ('what', 72382),\n",
       " ('do', 71217),\n",
       " ('now', 65151),\n",
       " ('they', 64454),\n",
       " ('baby', 64044),\n",
       " ('if', 62639),\n",
       " ('can', 62202),\n",
       " ('out', 60836),\n",
       " ('she', 56029),\n",
       " ('was', 55450),\n",
       " ('go', 54723),\n",
       " ('down', 50390),\n",
       " ('one', 49764),\n",
       " (\"ain't\", 49189),\n",
       " ('see', 46677),\n",
       " ('time', 46552),\n",
       " ('never', 46423),\n",
       " (\"you're\", 45625),\n",
       " ('want', 44823),\n",
       " (\"can't\", 44071),\n",
       " ('come', 43327),\n",
       " ('make', 42673),\n",
       " ('her', 42253),\n",
       " ('back', 42200),\n",
       " ('let', 41461),\n",
       " ('say', 40491),\n",
       " ('from', 40344),\n",
       " ('not', 40302),\n",
       " ('at', 39561),\n",
       " ('have', 38958),\n",
       " ('he', 38950),\n",
       " ('way', 37950),\n",
       " ('how', 37686),\n",
       " ('are', 37119),\n",
       " ('nigga', 36970),\n",
       " ('take', 35818),\n",
       " ('wanna', 35707),\n",
       " ('right', 34046),\n",
       " ('as', 32330),\n",
       " ('need', 32242),\n",
       " (\"that's\", 31961),\n",
       " (\"i'll\", 31782),\n",
       " ('will', 31692),\n",
       " ('tell', 31016),\n",
       " ('been', 30264),\n",
       " ('man', 29793),\n",
       " ('niggas', 29783),\n",
       " ('feel', 29711),\n",
       " ('girl', 29477),\n",
       " (\"'cause\", 28818),\n",
       " ('shit', 28140),\n",
       " ('here', 28086),\n",
       " ('life', 27619),\n",
       " ('there', 27600),\n",
       " ('about', 27409),\n",
       " ('some', 27226),\n",
       " ('gonna', 26583),\n",
       " ('fuck', 26308),\n",
       " ('more', 26095),\n",
       " ('give', 25812),\n",
       " ('good', 25673),\n",
       " ('then', 25658),\n",
       " ('keep', 25169)]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common rap word\n",
    "print(\"Most Common Rap Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='rap','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Rock Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 809641),\n",
       " ('i', 656646),\n",
       " ('you', 653628),\n",
       " ('to', 475124),\n",
       " ('and', 469616),\n",
       " ('a', 402732),\n",
       " ('me', 296258),\n",
       " ('in', 266901),\n",
       " ('my', 249782),\n",
       " ('it', 235627),\n",
       " ('of', 230703),\n",
       " ('your', 198446),\n",
       " ('on', 176022),\n",
       " ('that', 173440),\n",
       " ('all', 162517),\n",
       " (\"i'm\", 154407),\n",
       " ('is', 150609),\n",
       " ('for', 140552),\n",
       " ('be', 134327),\n",
       " ('we', 125082),\n",
       " ('so', 124034),\n",
       " (\"don't\", 123951),\n",
       " ('but', 118421),\n",
       " (\"it's\", 113100),\n",
       " ('no', 111772),\n",
       " ('know', 109285),\n",
       " ('just', 104807),\n",
       " ('this', 103436),\n",
       " ('with', 101522),\n",
       " ('love', 101394),\n",
       " ('when', 92817),\n",
       " ('like', 92004),\n",
       " ('what', 91059),\n",
       " ('up', 84689),\n",
       " ('can', 82579),\n",
       " ('now', 81113),\n",
       " ('oh', 78889),\n",
       " ('down', 77905),\n",
       " ('out', 77462),\n",
       " ('do', 75551),\n",
       " (\"you're\", 74413),\n",
       " ('got', 73020),\n",
       " ('time', 72931),\n",
       " ('are', 72930),\n",
       " ('if', 72181),\n",
       " ('was', 69977),\n",
       " ('never', 67879),\n",
       " ('one', 67405),\n",
       " ('will', 67286),\n",
       " ('see', 65942),\n",
       " ('not', 64567),\n",
       " ('get', 64453),\n",
       " ('go', 64414),\n",
       " ('have', 59884),\n",
       " ('she', 59340),\n",
       " ('they', 58957),\n",
       " ('want', 58198),\n",
       " ('from', 58121),\n",
       " (\"can't\", 57385),\n",
       " ('way', 53678),\n",
       " ('come', 53246),\n",
       " ('at', 51115),\n",
       " ('say', 49683),\n",
       " ('as', 49314),\n",
       " ('take', 49273),\n",
       " ('away', 49172),\n",
       " ('her', 47960),\n",
       " (\"i'll\", 47800),\n",
       " ('back', 47556),\n",
       " ('he', 46088),\n",
       " ('yeah', 45528),\n",
       " ('there', 45418),\n",
       " (\"i've\", 44410),\n",
       " ('let', 44345),\n",
       " ('make', 43752),\n",
       " ('baby', 42915),\n",
       " ('gonna', 42548),\n",
       " ('here', 40612),\n",
       " ('life', 40523),\n",
       " ('feel', 40431),\n",
       " ('how', 40373),\n",
       " ('been', 39765),\n",
       " ('could', 38762),\n",
       " ('by', 37370),\n",
       " ('where', 36705),\n",
       " ('day', 36438),\n",
       " ('night', 35991),\n",
       " ('right', 35691),\n",
       " ('more', 35436),\n",
       " ('our', 35258),\n",
       " (\"there's\", 35198),\n",
       " ('well', 34792),\n",
       " ('heart', 33398),\n",
       " ('need', 33335),\n",
       " ('tell', 33269),\n",
       " ('too', 33034),\n",
       " ('through', 33004),\n",
       " ('again', 32942),\n",
       " ('world', 32784),\n",
       " ('think', 31084)]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common rock word\n",
    "print(\"Most Common Rock Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Rock','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common R&B Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('you', 32919),\n",
       " ('i', 29075),\n",
       " ('the', 23521),\n",
       " ('to', 18520),\n",
       " ('and', 16474),\n",
       " ('me', 13747),\n",
       " ('a', 13241),\n",
       " ('my', 10566),\n",
       " ('it', 10561),\n",
       " ('in', 8419),\n",
       " ('that', 7579),\n",
       " ('your', 7550),\n",
       " ('love', 7292),\n",
       " ('of', 6988),\n",
       " ('on', 6863),\n",
       " (\"i'm\", 6468),\n",
       " ('be', 6041),\n",
       " ('all', 5877),\n",
       " (\"don't\", 5796),\n",
       " ('for', 5545),\n",
       " ('baby', 5439),\n",
       " ('know', 5260),\n",
       " ('is', 5249),\n",
       " ('we', 5234),\n",
       " ('so', 5213),\n",
       " ('just', 4964),\n",
       " ('no', 4735),\n",
       " ('oh', 4672),\n",
       " ('do', 4518),\n",
       " ('but', 4450),\n",
       " ('with', 3970),\n",
       " ('when', 3942),\n",
       " (\"it's\", 3891),\n",
       " ('what', 3698),\n",
       " ('can', 3669),\n",
       " ('got', 3618),\n",
       " ('like', 3492),\n",
       " ('this', 3435),\n",
       " ('now', 3418),\n",
       " ('get', 3147),\n",
       " ('want', 3124),\n",
       " ('if', 3108),\n",
       " (\"you're\", 3085),\n",
       " ('yeah', 3048),\n",
       " ('up', 2820),\n",
       " ('time', 2726),\n",
       " (\"can't\", 2693),\n",
       " ('go', 2682),\n",
       " ('one', 2640),\n",
       " ('was', 2596),\n",
       " ('have', 2521),\n",
       " ('never', 2495),\n",
       " ('see', 2461),\n",
       " ('make', 2403),\n",
       " ('out', 2397),\n",
       " ('let', 2373),\n",
       " ('down', 2319),\n",
       " ('not', 2272),\n",
       " ('girl', 2257),\n",
       " ('come', 2241),\n",
       " ('say', 2223),\n",
       " ('will', 2182),\n",
       " ('are', 2146),\n",
       " ('they', 2009),\n",
       " ('back', 2006),\n",
       " ('way', 1973),\n",
       " ('she', 1950),\n",
       " (\"i'll\", 1940),\n",
       " ('gonna', 1932),\n",
       " ('take', 1905),\n",
       " ('right', 1844),\n",
       " ('how', 1826),\n",
       " ('at', 1787),\n",
       " ('he', 1775),\n",
       " ('tell', 1720),\n",
       " ('need', 1702),\n",
       " ('been', 1701),\n",
       " ('there', 1696),\n",
       " ('from', 1685),\n",
       " ('her', 1677),\n",
       " ('heart', 1659),\n",
       " ('feel', 1639),\n",
       " ('by', 1633),\n",
       " (\"ain't\", 1630),\n",
       " ('about', 1559),\n",
       " ('life', 1550),\n",
       " ('as', 1545),\n",
       " (\"i've\", 1536),\n",
       " ('more', 1520),\n",
       " (\"'cause\", 1500),\n",
       " ('wanna', 1499),\n",
       " ('man', 1463),\n",
       " (\"that's\", 1436),\n",
       " ('could', 1430),\n",
       " ('night', 1413),\n",
       " ('day', 1410),\n",
       " ('away', 1409),\n",
       " ('our', 1391),\n",
       " ('good', 1380),\n",
       " ('think', 1374)]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common R&B word\n",
    "print(\"Most Common R&B Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='R&B','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Hip-Hop Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 450218),\n",
       " ('i', 353479),\n",
       " ('you', 308374),\n",
       " ('a', 250858),\n",
       " ('and', 234834),\n",
       " ('to', 223949),\n",
       " ('my', 170342),\n",
       " ('it', 159682),\n",
       " ('me', 154743),\n",
       " ('in', 147375),\n",
       " (\"i'm\", 120826),\n",
       " ('on', 118469),\n",
       " ('that', 111389),\n",
       " ('like', 98684),\n",
       " ('your', 88240),\n",
       " ('of', 86955),\n",
       " ('we', 84634),\n",
       " ('with', 82997),\n",
       " ('up', 82735),\n",
       " ('get', 74615),\n",
       " ('for', 73990),\n",
       " ('is', 71635),\n",
       " ('all', 68589),\n",
       " ('got', 67075),\n",
       " ('be', 65619),\n",
       " ('so', 63652),\n",
       " ('this', 62253),\n",
       " (\"don't\", 61965),\n",
       " ('know', 61145),\n",
       " ('but', 59970),\n",
       " ('they', 57558),\n",
       " ('no', 56305),\n",
       " ('what', 50052),\n",
       " ('just', 49478),\n",
       " ('when', 49221),\n",
       " (\"it's\", 48499),\n",
       " ('out', 47113),\n",
       " ('do', 43763),\n",
       " (\"ain't\", 43180),\n",
       " ('now', 42529),\n",
       " ('nigga', 41514),\n",
       " ('if', 40561),\n",
       " ('can', 37785),\n",
       " ('shit', 35739),\n",
       " ('was', 35513),\n",
       " ('go', 34011),\n",
       " ('see', 33939),\n",
       " ('she', 33621),\n",
       " ('from', 32929),\n",
       " ('down', 32711),\n",
       " ('love', 31559),\n",
       " ('yeah', 31334),\n",
       " ('back', 30983),\n",
       " ('niggas', 29988),\n",
       " ('ya', 29538),\n",
       " ('make', 29401),\n",
       " ('at', 29294),\n",
       " ('one', 28753),\n",
       " ('fuck', 28670),\n",
       " ('man', 27415),\n",
       " ('want', 26817),\n",
       " ('come', 25581),\n",
       " ('her', 25210),\n",
       " ('how', 25019),\n",
       " ('never', 24842),\n",
       " ('time', 24566),\n",
       " (\"that's\", 24496),\n",
       " ('let', 24425),\n",
       " ('them', 24373),\n",
       " ('baby', 24128),\n",
       " ('yo', 23544),\n",
       " ('say', 23485),\n",
       " ('some', 23453),\n",
       " (\"can't\", 22940),\n",
       " ('he', 22812),\n",
       " ('not', 21929),\n",
       " ('cause', 21752),\n",
       " ('bitch', 21751),\n",
       " ('take', 21712),\n",
       " ('wanna', 21553),\n",
       " ('off', 21357),\n",
       " ('then', 20963),\n",
       " ('right', 20910),\n",
       " ('oh', 20764),\n",
       " ('girl', 20647),\n",
       " ('as', 20193),\n",
       " ('or', 20119),\n",
       " ('money', 19927),\n",
       " ('have', 19731),\n",
       " ('way', 19369),\n",
       " ('about', 18940),\n",
       " ('tell', 18824),\n",
       " ('keep', 18723),\n",
       " ('who', 18103),\n",
       " ('here', 17612),\n",
       " ('these', 17519),\n",
       " ('life', 17461),\n",
       " ('need', 17214),\n",
       " (\"'cause\", 17211),\n",
       " ('la', 16617)]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Hip-Hop word\n",
    "print(\"Most Common Hip-Hop Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Hip-Hop','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Indie Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 24740),\n",
       " ('i', 21569),\n",
       " ('you', 19957),\n",
       " ('and', 15990),\n",
       " ('to', 13959),\n",
       " ('a', 11280),\n",
       " ('me', 8709),\n",
       " ('in', 8265),\n",
       " ('my', 8157),\n",
       " ('it', 7041),\n",
       " ('of', 6594),\n",
       " ('your', 6177),\n",
       " ('that', 5677),\n",
       " ('all', 5071),\n",
       " ('on', 4788),\n",
       " (\"i'm\", 4507),\n",
       " ('for', 4348),\n",
       " ('be', 4220),\n",
       " ('is', 4205),\n",
       " ('we', 4125),\n",
       " ('but', 3907),\n",
       " (\"don't\", 3814),\n",
       " ('know', 3757),\n",
       " ('so', 3727),\n",
       " ('love', 3190),\n",
       " (\"it's\", 3148),\n",
       " ('with', 3143),\n",
       " ('like', 3060),\n",
       " ('just', 3033),\n",
       " ('when', 3030),\n",
       " ('no', 2893),\n",
       " ('this', 2865),\n",
       " ('up', 2861),\n",
       " ('what', 2823),\n",
       " ('oh', 2813),\n",
       " ('if', 2605),\n",
       " ('are', 2470),\n",
       " ('out', 2388),\n",
       " ('can', 2307),\n",
       " ('was', 2276),\n",
       " ('will', 2262),\n",
       " (\"you're\", 2213),\n",
       " ('not', 2211),\n",
       " ('now', 2181),\n",
       " ('have', 2138),\n",
       " ('down', 2119),\n",
       " ('do', 2119),\n",
       " ('go', 2085),\n",
       " ('time', 2045),\n",
       " ('they', 2012),\n",
       " ('from', 1932),\n",
       " ('never', 1896),\n",
       " ('get', 1850),\n",
       " ('one', 1784),\n",
       " ('come', 1725),\n",
       " ('at', 1672),\n",
       " ('see', 1670),\n",
       " ('as', 1633),\n",
       " (\"i'll\", 1619),\n",
       " ('got', 1618),\n",
       " ('say', 1571),\n",
       " ('way', 1489),\n",
       " ('let', 1489),\n",
       " ('want', 1480),\n",
       " (\"i've\", 1458),\n",
       " ('there', 1440),\n",
       " ('take', 1432),\n",
       " ('she', 1418),\n",
       " ('how', 1381),\n",
       " ('been', 1362),\n",
       " ('back', 1357),\n",
       " (\"can't\", 1349),\n",
       " ('where', 1340),\n",
       " ('could', 1323),\n",
       " ('away', 1261),\n",
       " ('her', 1258),\n",
       " ('here', 1239),\n",
       " ('our', 1225),\n",
       " ('make', 1216),\n",
       " ('feel', 1166),\n",
       " ('by', 1104),\n",
       " (\"there's\", 1099),\n",
       " ('heart', 1097),\n",
       " ('he', 1085),\n",
       " ('gonna', 1063),\n",
       " ('only', 1048),\n",
       " ('day', 1046),\n",
       " ('through', 1029),\n",
       " ('too', 1026),\n",
       " ('home', 1003),\n",
       " ('life', 996),\n",
       " (\"won't\", 994),\n",
       " ('right', 980),\n",
       " ('am', 980),\n",
       " ('were', 977),\n",
       " ('think', 972),\n",
       " ('still', 970),\n",
       " ('well', 961),\n",
       " ('la', 960),\n",
       " ('night', 953)]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Indie word\n",
    "print(\"Most Common Indie Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Indie','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Country Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 103946),\n",
       " ('i', 91420),\n",
       " ('you', 75324),\n",
       " ('and', 73807),\n",
       " ('a', 62347),\n",
       " ('to', 60305),\n",
       " ('me', 39825),\n",
       " ('my', 37632),\n",
       " ('in', 36284),\n",
       " ('of', 30986),\n",
       " ('that', 29505),\n",
       " ('it', 26451),\n",
       " ('on', 25861),\n",
       " ('love', 20291),\n",
       " ('your', 19921),\n",
       " ('but', 19511),\n",
       " ('be', 19131),\n",
       " (\"i'm\", 19047),\n",
       " ('all', 18788),\n",
       " ('for', 18742),\n",
       " ('just', 16255),\n",
       " ('when', 15497),\n",
       " ('is', 15311),\n",
       " ('with', 14174),\n",
       " (\"don't\", 13762),\n",
       " ('so', 13720),\n",
       " ('we', 13520),\n",
       " ('know', 12836),\n",
       " ('like', 12678),\n",
       " ('was', 12618),\n",
       " (\"it's\", 12256),\n",
       " ('if', 11755),\n",
       " ('this', 10946),\n",
       " ('she', 10912),\n",
       " ('up', 10692),\n",
       " ('her', 10548),\n",
       " ('no', 10455),\n",
       " ('he', 10382),\n",
       " ('down', 10304),\n",
       " ('one', 10000),\n",
       " ('can', 9934),\n",
       " ('got', 9531),\n",
       " ('what', 9493),\n",
       " ('time', 9379),\n",
       " ('now', 9061),\n",
       " ('oh', 8808),\n",
       " ('go', 8795),\n",
       " ('out', 8715),\n",
       " ('as', 8663),\n",
       " (\"i'll\", 8599),\n",
       " ('do', 8576),\n",
       " ('never', 8525),\n",
       " ('have', 8361),\n",
       " (\"you're\", 8129),\n",
       " ('way', 7788),\n",
       " ('back', 7770),\n",
       " ('from', 7676),\n",
       " (\"i've\", 7544),\n",
       " ('at', 7498),\n",
       " ('there', 7438),\n",
       " ('they', 7432),\n",
       " ('heart', 7311),\n",
       " ('get', 7261),\n",
       " ('see', 7065),\n",
       " ('little', 6986),\n",
       " ('are', 6898),\n",
       " ('will', 6881),\n",
       " ('not', 6678),\n",
       " (\"can't\", 6496),\n",
       " ('his', 6408),\n",
       " ('well', 6361),\n",
       " ('come', 6358),\n",
       " ('gonna', 6314),\n",
       " ('where', 6165),\n",
       " ('been', 6164),\n",
       " ('say', 6145),\n",
       " ('how', 6123),\n",
       " ('night', 5804),\n",
       " ('could', 5748),\n",
       " ('by', 5732),\n",
       " ('let', 5694),\n",
       " ('old', 5680),\n",
       " ('baby', 5643),\n",
       " (\"ain't\", 5563),\n",
       " ('take', 5457),\n",
       " ('day', 5450),\n",
       " ('home', 5386),\n",
       " ('here', 5368),\n",
       " ('away', 5295),\n",
       " (\"there's\", 5240),\n",
       " ('long', 5228),\n",
       " ('make', 5096),\n",
       " (\"that's\", 5001),\n",
       " ('man', 4976),\n",
       " ('good', 4928),\n",
       " ('right', 4910),\n",
       " ('then', 4905),\n",
       " ('had', 4845),\n",
       " ('want', 4791),\n",
       " ('more', 4786)]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Country word\n",
    "print(\"Most Common Country Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Country','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Electronic Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 54787),\n",
       " ('you', 53148),\n",
       " ('i', 48938),\n",
       " ('to', 32062),\n",
       " ('and', 30006),\n",
       " ('me', 27230),\n",
       " ('a', 27126),\n",
       " ('it', 21646),\n",
       " ('my', 19177),\n",
       " ('in', 18843),\n",
       " ('your', 15685),\n",
       " ('of', 13780),\n",
       " ('is', 12497),\n",
       " ('on', 12385),\n",
       " ('we', 12359),\n",
       " (\"i'm\", 11794),\n",
       " ('that', 11301),\n",
       " ('all', 11087),\n",
       " ('love', 10569),\n",
       " ('be', 10221),\n",
       " ('for', 9759),\n",
       " ('so', 9459),\n",
       " (\"don't\", 8830),\n",
       " ('no', 8691),\n",
       " ('this', 8055),\n",
       " ('know', 7853),\n",
       " ('up', 7845),\n",
       " ('with', 7589),\n",
       " ('like', 7422),\n",
       " ('but', 7379),\n",
       " ('just', 7364),\n",
       " ('can', 7066),\n",
       " ('what', 7047),\n",
       " ('do', 7026),\n",
       " (\"it's\", 6967),\n",
       " ('oh', 6924),\n",
       " ('when', 6670),\n",
       " ('now', 6546),\n",
       " ('get', 6308),\n",
       " ('are', 6224),\n",
       " ('go', 5989),\n",
       " ('come', 5375),\n",
       " (\"you're\", 5177),\n",
       " ('will', 5166),\n",
       " ('see', 5111),\n",
       " ('want', 5078),\n",
       " ('if', 4995),\n",
       " ('got', 4964),\n",
       " ('out', 4921),\n",
       " ('down', 4916),\n",
       " ('feel', 4864),\n",
       " ('time', 4858),\n",
       " ('one', 4823),\n",
       " (\"can't\", 4656),\n",
       " ('never', 4611),\n",
       " ('have', 4295),\n",
       " ('not', 4164),\n",
       " ('make', 4106),\n",
       " ('take', 4059),\n",
       " ('let', 4009),\n",
       " ('baby', 3892),\n",
       " ('from', 3862),\n",
       " ('was', 3852),\n",
       " ('yeah', 3799),\n",
       " ('way', 3730),\n",
       " ('say', 3588),\n",
       " ('wanna', 3480),\n",
       " ('back', 3412),\n",
       " (\"i'll\", 3298),\n",
       " ('life', 3220),\n",
       " ('need', 3167),\n",
       " ('they', 3167),\n",
       " ('here', 3149),\n",
       " ('away', 3136),\n",
       " ('there', 3134),\n",
       " ('she', 3121),\n",
       " ('give', 3056),\n",
       " ('right', 3045),\n",
       " ('night', 3032),\n",
       " ('how', 2947),\n",
       " ('world', 2891),\n",
       " ('gonna', 2817),\n",
       " ('at', 2790),\n",
       " ('as', 2783),\n",
       " ('heart', 2760),\n",
       " ('la', 2714),\n",
       " ('tell', 2685),\n",
       " ('more', 2675),\n",
       " ('could', 2668),\n",
       " ('where', 2560),\n",
       " (\"i've\", 2288),\n",
       " (\"we're\", 2287),\n",
       " ('by', 2263),\n",
       " ('too', 2259),\n",
       " ('over', 2259),\n",
       " ('day', 2241),\n",
       " ('our', 2232),\n",
       " ('been', 2175),\n",
       " ('keep', 2157),\n",
       " ('her', 2140)]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Electronic word\n",
    "print(\"Most Common Electronic Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Electronic','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Folk Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 18186),\n",
       " ('and', 10907),\n",
       " ('i', 10343),\n",
       " ('a', 8933),\n",
       " ('to', 7949),\n",
       " ('you', 7897),\n",
       " ('in', 5596),\n",
       " ('of', 5079),\n",
       " ('my', 4758),\n",
       " ('me', 4652),\n",
       " ('on', 3119),\n",
       " ('it', 3104),\n",
       " ('is', 3008),\n",
       " ('that', 2914),\n",
       " ('for', 2730),\n",
       " ('your', 2625),\n",
       " ('all', 2434),\n",
       " ('we', 2210),\n",
       " ('be', 2141),\n",
       " ('with', 1952),\n",
       " ('but', 1936),\n",
       " ('so', 1825),\n",
       " ('love', 1705),\n",
       " ('when', 1676),\n",
       " ('was', 1663),\n",
       " ('no', 1616),\n",
       " (\"i'm\", 1590),\n",
       " ('are', 1533),\n",
       " ('will', 1492),\n",
       " ('this', 1428),\n",
       " ('he', 1390),\n",
       " ('they', 1367),\n",
       " ('go', 1366),\n",
       " ('like', 1351),\n",
       " ('oh', 1344),\n",
       " ('from', 1320),\n",
       " (\"it's\", 1295),\n",
       " ('as', 1222),\n",
       " ('know', 1222),\n",
       " ('down', 1193),\n",
       " ('her', 1172),\n",
       " ('if', 1159),\n",
       " ('not', 1143),\n",
       " ('can', 1131),\n",
       " ('she', 1118),\n",
       " ('now', 1117),\n",
       " ('just', 1098),\n",
       " ('have', 1076),\n",
       " ('by', 1075),\n",
       " ('up', 1071),\n",
       " (\"don't\", 1058),\n",
       " ('out', 1051),\n",
       " ('what', 1029),\n",
       " ('at', 1027),\n",
       " ('never', 1025),\n",
       " ('there', 996),\n",
       " ('his', 992),\n",
       " ('see', 990),\n",
       " ('do', 988),\n",
       " ('time', 987),\n",
       " ('one', 918),\n",
       " ('an', 891),\n",
       " ('where', 858),\n",
       " ('come', 843),\n",
       " ('away', 832),\n",
       " ('day', 815),\n",
       " ('our', 784),\n",
       " ('way', 783),\n",
       " ('que', 758),\n",
       " ('de', 729),\n",
       " (\"i'll\", 725),\n",
       " ('got', 724),\n",
       " ('heart', 708),\n",
       " (\"you're\", 667),\n",
       " ('let', 666),\n",
       " ('get', 659),\n",
       " ('night', 631),\n",
       " ('back', 624),\n",
       " ('who', 622),\n",
       " ('how', 618),\n",
       " ('then', 616),\n",
       " (\"i've\", 616),\n",
       " ('were', 607),\n",
       " ('man', 595),\n",
       " ('take', 594),\n",
       " ('la', 588),\n",
       " (\"can't\", 587),\n",
       " ('through', 584),\n",
       " ('say', 582),\n",
       " ('us', 568),\n",
       " ('their', 567),\n",
       " ('been', 566),\n",
       " ('am', 556),\n",
       " ('s', 555),\n",
       " ('or', 549),\n",
       " ('him', 549),\n",
       " ('want', 538),\n",
       " ('more', 537),\n",
       " ('old', 529),\n",
       " ('could', 525)]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Folk word\n",
    "print(\"Most Common Folk Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Folk','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Common Jazz Words: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('the', 46689),\n",
       " ('you', 43949),\n",
       " ('i', 42839),\n",
       " ('and', 30622),\n",
       " ('to', 30230),\n",
       " ('a', 28193),\n",
       " ('me', 22346),\n",
       " ('my', 20166),\n",
       " ('in', 17312),\n",
       " ('of', 14219),\n",
       " ('love', 13746),\n",
       " ('that', 13104),\n",
       " ('it', 12830),\n",
       " ('your', 11238),\n",
       " ('be', 11117),\n",
       " ('is', 10435),\n",
       " ('on', 10062),\n",
       " (\"i'm\", 9656),\n",
       " ('for', 9404),\n",
       " ('all', 9216),\n",
       " ('so', 8624),\n",
       " ('but', 8129),\n",
       " ('with', 7745),\n",
       " ('when', 7724),\n",
       " ('just', 6934),\n",
       " ('know', 6919),\n",
       " (\"don't\", 6876),\n",
       " ('no', 6809),\n",
       " ('do', 6258),\n",
       " ('we', 6153),\n",
       " (\"it's\", 5908),\n",
       " ('oh', 5907),\n",
       " ('if', 5587),\n",
       " ('like', 5569),\n",
       " ('what', 5543),\n",
       " ('can', 5403),\n",
       " ('got', 5237),\n",
       " ('this', 4921),\n",
       " ('baby', 4736),\n",
       " ('now', 4653),\n",
       " ('was', 4577),\n",
       " (\"you're\", 4458),\n",
       " ('have', 4388),\n",
       " ('up', 4380),\n",
       " ('one', 4301),\n",
       " ('are', 4167),\n",
       " ('never', 4167),\n",
       " ('go', 4165),\n",
       " ('will', 4098),\n",
       " ('come', 4031),\n",
       " ('see', 3796),\n",
       " ('heart', 3795),\n",
       " ('get', 3786),\n",
       " ('time', 3781),\n",
       " ('as', 3769),\n",
       " ('say', 3708),\n",
       " ('he', 3704),\n",
       " ('down', 3696),\n",
       " (\"i'll\", 3630),\n",
       " ('they', 3515),\n",
       " ('out', 3497),\n",
       " ('way', 3482),\n",
       " (\"can't\", 3438),\n",
       " ('how', 3424),\n",
       " ('from', 3353),\n",
       " ('not', 3350),\n",
       " ('let', 3285),\n",
       " ('she', 3283),\n",
       " ('at', 3227),\n",
       " ('day', 3204),\n",
       " ('there', 3139),\n",
       " ('make', 3018),\n",
       " ('want', 2969),\n",
       " ('her', 2881),\n",
       " ('night', 2755),\n",
       " ('gonna', 2739),\n",
       " (\"i've\", 2704),\n",
       " ('take', 2458),\n",
       " ('little', 2451),\n",
       " ('tell', 2446),\n",
       " ('too', 2435),\n",
       " ('away', 2404),\n",
       " ('or', 2400),\n",
       " ('could', 2388),\n",
       " ('by', 2361),\n",
       " ('why', 2353),\n",
       " ('de', 2338),\n",
       " ('back', 2289),\n",
       " ('more', 2264),\n",
       " ('where', 2252),\n",
       " ('here', 2209),\n",
       " (\"that's\", 2181),\n",
       " ('well', 2169),\n",
       " ('good', 2166),\n",
       " ('man', 2166),\n",
       " ('then', 2146),\n",
       " ('la', 2141),\n",
       " ('feel', 2137),\n",
       " ('life', 2130),\n",
       " ('right', 2127)]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#most common Jazz word\n",
    "print(\"Most Common Jazz Words: \")\n",
    "Counter(\" \".join(dta.loc[dta['genre']=='Jazz','lyrics']).split()).most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metrics(text, genre):\n",
    "    #number of distinct words in a lyric\n",
    "    counter = Counter(text[text['genre'] == genre]['lyrics'])\n",
    "    distinct_words = len(counter)\n",
    "\n",
    "    #average number of tokens in a lyric\n",
    "    len_toks = sum(text[text['genre'] == genre]['t-lyric'].str.len())/len(text[text['genre']== genre]['t-lyric'])\n",
    "  \n",
    "    \n",
    "    #average line length \n",
    "    subset = text[text['genre'] == genre]\n",
    "    songs = list(subset[subset['lyrics'].notnull()]['lyrics'].str.split(pat = \"\\n\"))#songs\n",
    "\n",
    "    song_line_len = []\n",
    "    for song in songs:\n",
    "        line_lens = [] \n",
    "        for line in song:\n",
    "            line_lens.append(len(line.split(\" \")))\n",
    "        avg_line_per_song = sum(line_lens)/len(song)\n",
    "        song_line_len.append(avg_line_per_song)\n",
    "    avg_line_len = sum(song_line_len)/len(songs)\n",
    "    \n",
    "    \n",
    "    #average word length\n",
    "    song_line_len = []\n",
    "    for song in songs:\n",
    "        line_lens = []\n",
    "        for line in song:\n",
    "            line_lens.append(len(line)/len(line.split(\" \")))\n",
    "        avg_line_per_song = sum(line_lens)/len(song)\n",
    "        song_line_len.append(avg_line_per_song)\n",
    "    avg_word_len = sum(song_line_len)/len(songs)\n",
    "\n",
    "    \n",
    "    return distinct_words, len_toks, avg_word_len, avg_line_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22446, 1150.7218741199663, 5.475789991276703, 8.225387365296852)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'Metal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric(dta, 'Rock')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6812, 649.7764040356914, 5.001229703760138, 7.017426795587943)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'Jazz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(47574, 2413.218287378423, 4.513449685931287, 7.014357455270661)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'rap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3024, 859.9513258897418, 4.954726674096404, 7.103142997146679)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'Indie')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1999, 1004.5695772909596, 5.4225414676646935, 6.641886005213627)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'Folk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7044, 756.0140080222154, 5.221263824531677, 5.9708978039941)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'Electronic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3156, 1010.1563605728728, 4.8092544596103615, 8.148398223844897)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta, 'R&B')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22613, 2795.0239952892684, 4.955525258445672, 9.301993085782962)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics(dta,'Hip-Hop')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}