proposal_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": "### Few Hit Wonders versus Consistent Hitters\n##### Joseph Scarpa (jcs140)",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "644549be",
    "execution_start": 1634512206835,
    "cell_id": "00000-4503efd3-9073-4776-9c00-0bf6f972c57f",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "2a5bb180",
    "execution_start": 1638642118737,
    "execution_millis": 2103,
    "is_code_hidden": false,
    "cell_id": "00001-f7a04732-10a5-4e2c-adff-4125db866e6e",
    "deepnote_cell_type": "code"
   },
   "source": "# Imports (update throughout as necessary)\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport math\nimport matplotlib.pyplot as plt\npd.options.mode.chained_assignment = None\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import plot_confusion_matrix\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import r2_score\nimport matplotlib.pyplot as plt # just to format confusion matrix\n\nfrom sklearn.feature_selection import SequentialFeatureSelector\nfrom sklearn.neighbors import KNeighborsClassifier\n\nimport re\n\nimport scipy\nfrom scipy import stats",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "#### Data loading and cleaning\n\nNotes:\n* No null values found\n* No duplicate values found",
   "metadata": {
    "tags": [],
    "cell_id": "00002-87cd9e5e-fada-4450-86b2-13be7052ad52",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "1796852b",
    "execution_start": 1638642120843,
    "execution_millis": 228,
    "is_code_hidden": false,
    "cell_id": "00003-887177e6-ad9b-4ce9-bf39-b9f3753e42bb",
    "deepnote_cell_type": "code"
   },
   "source": "# Data loading\ndata10 = pd.read_csv('./datasets/dataset-of-10s.csv')\ndata00 = pd.read_csv('./datasets/dataset-of-00s.csv')\ndata90 = pd.read_csv('./datasets/dataset-of-90s.csv')\ndata80 = pd.read_csv('./datasets/dataset-of-80s.csv')\ndata70 = pd.read_csv('./datasets/dataset-of-70s.csv')\ndata60 = pd.read_csv('./datasets/dataset-of-60s.csv')",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d1ca17dc",
    "execution_start": 1638642121084,
    "execution_millis": 68,
    "is_code_hidden": false,
    "cell_id": "00004-96c0becc-21ad-4816-bbb8-29e7431a1ca7",
    "deepnote_cell_type": "code"
   },
   "source": "# Data cleaning\n\n# Drop all na values\ndata10 = data10.dropna()\ndata00 = data00.dropna()\ndata90 = data90.dropna()\ndata80 = data80.dropna()\ndata70 = data70.dropna()\ndata60 = data60.dropna()\n\n\n# Add a decades column to each dataset (will be helpful when creating cumulative dataset)\ndata10['decade'] = [2010]*len(data10)\ndata00['decade'] = [2000]*len(data00)\ndata90['decade'] = [1990]*len(data90)\ndata80['decade'] = [1980]*len(data80)\ndata70['decade'] = [1970]*len(data70)\ndata60['decade'] = [1960]*len(data60)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "57d78fe",
    "execution_start": 1638642121159,
    "execution_millis": 230,
    "is_code_hidden": false,
    "is_output_hidden": false,
    "cell_id": "00005-b4fe0f07-cb59-463a-9124-411dc04983a2",
    "deepnote_cell_type": "code"
   },
   "source": "# Check for duplicates in dataset\nmerged = data10.merge(data00,on=['track','artist'])\nmerged = merged.merge(data90,on=['track','artist'])\nmerged = merged.merge(data80,on=['track','artist'])\nmerged = merged.merge(data70,on=['track','artist'])\nmerged = merged.merge(data60,on=['track','artist'])\nmerged\n# NOTE: No duplicates found\n\n# Create cumulative dataset (from songs across all decades)\ndata = data10.append(data00,ignore_index=True).append(data90,ignore_index=True).append(data80,ignore_index=True).append(data70,ignore_index=True).append(data60,ignore_index=True)\ndata.head()\ndata.tail()",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 4,
     "data": {
      "application/vnd.deepnote.dataframe.v2+json": {
       "row_count": 5,
       "column_count": 20,
       "columns": [
        {
         "name": "track",
         "dtype": "object",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "categories": [
           {
            "name": "Le ire di Ursus",
            "count": 1
           },
           {
            "name": "A-B-C",
            "count": 1
           },
           {
            "name": "3 others",
            "count": 3
           }
          ]
         }
        },
        {
         "name": "artist",
         "dtype": "object",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "categories": [
           {
            "name": "Giampiero Boneschi",
            "count": 1
           },
           {
            "name": "John Zacherle",
            "count": 1
           },
           {
            "name": "3 others",
            "count": 3
           }
          ]
         }
        },
        {
         "name": "uri",
         "dtype": "object",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "categories": [
           {
            "name": "spotify:track:3dVgolCVuNgecU5jovpc45",
            "count": 1
           },
           {
            "name": "spotify:track:0zXhzKf4raEYstebL7L42C",
            "count": 1
           },
           {
            "name": "3 others",
            "count": 3
           }
          ]
         }
        },
        {
         "name": "danceability",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.361",
          "max": "0.716",
          "histogram": [
           {
            "bin_start": 0.361,
            "bin_end": 0.39649999999999996,
            "count": 1
           },
           {
            "bin_start": 0.39649999999999996,
            "bin_end": 0.432,
            "count": 1
           },
           {
            "bin_start": 0.432,
            "bin_end": 0.46749999999999997,
            "count": 1
           },
           {
            "bin_start": 0.46749999999999997,
            "bin_end": 0.503,
            "count": 0
           },
           {
            "bin_start": 0.503,
            "bin_end": 0.5385,
            "count": 0
           },
           {
            "bin_start": 0.5385,
            "bin_end": 0.574,
            "count": 1
           },
           {
            "bin_start": 0.574,
            "bin_end": 0.6094999999999999,
            "count": 0
           },
           {
            "bin_start": 0.6094999999999999,
            "bin_end": 0.645,
            "count": 0
           },
           {
            "bin_start": 0.645,
            "bin_end": 0.6804999999999999,
            "count": 0
           },
           {
            "bin_start": 0.6804999999999999,
            "bin_end": 0.716,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "energy",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.0944",
          "max": "0.873",
          "histogram": [
           {
            "bin_start": 0.0944,
            "bin_end": 0.17226,
            "count": 1
           },
           {
            "bin_start": 0.17226,
            "bin_end": 0.25012,
            "count": 0
           },
           {
            "bin_start": 0.25012,
            "bin_end": 0.32798,
            "count": 1
           },
           {
            "bin_start": 0.32798,
            "bin_end": 0.40584,
            "count": 0
           },
           {
            "bin_start": 0.40584,
            "bin_end": 0.48369999999999996,
            "count": 0
           },
           {
            "bin_start": 0.48369999999999996,
            "bin_end": 0.5615600000000001,
            "count": 0
           },
           {
            "bin_start": 0.5615600000000001,
            "bin_end": 0.63942,
            "count": 0
           },
           {
            "bin_start": 0.63942,
            "bin_end": 0.71728,
            "count": 0
           },
           {
            "bin_start": 0.71728,
            "bin_end": 0.7951400000000001,
            "count": 2
           },
           {
            "bin_start": 0.7951400000000001,
            "bin_end": 0.873,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "key",
         "dtype": "int64",
         "stats": {
          "unique_count": 4,
          "nan_count": 0,
          "min": "3",
          "max": "10",
          "histogram": [
           {
            "bin_start": 3,
            "bin_end": 3.7,
            "count": 1
           },
           {
            "bin_start": 3.7,
            "bin_end": 4.4,
            "count": 1
           },
           {
            "bin_start": 4.4,
            "bin_end": 5.1,
            "count": 1
           },
           {
            "bin_start": 5.1,
            "bin_end": 5.8,
            "count": 0
           },
           {
            "bin_start": 5.8,
            "bin_end": 6.5,
            "count": 0
           },
           {
            "bin_start": 6.5,
            "bin_end": 7.199999999999999,
            "count": 0
           },
           {
            "bin_start": 7.199999999999999,
            "bin_end": 7.8999999999999995,
            "count": 0
           },
           {
            "bin_start": 7.8999999999999995,
            "bin_end": 8.6,
            "count": 0
           },
           {
            "bin_start": 8.6,
            "bin_end": 9.3,
            "count": 0
           },
           {
            "bin_start": 9.3,
            "bin_end": 10,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "loudness",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "-22.663",
          "max": "-5.727",
          "histogram": [
           {
            "bin_start": -22.663,
            "bin_end": -20.9694,
            "count": 1
           },
           {
            "bin_start": -20.9694,
            "bin_end": -19.2758,
            "count": 0
           },
           {
            "bin_start": -19.2758,
            "bin_end": -17.5822,
            "count": 0
           },
           {
            "bin_start": -17.5822,
            "bin_end": -15.8886,
            "count": 0
           },
           {
            "bin_start": -15.8886,
            "bin_end": -14.195,
            "count": 0
           },
           {
            "bin_start": -14.195,
            "bin_end": -12.5014,
            "count": 0
           },
           {
            "bin_start": -12.5014,
            "bin_end": -10.8078,
            "count": 1
           },
           {
            "bin_start": -10.8078,
            "bin_end": -9.1142,
            "count": 0
           },
           {
            "bin_start": -9.1142,
            "bin_end": -7.4206,
            "count": 2
           },
           {
            "bin_start": -7.4206,
            "bin_end": -5.727,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "mode",
         "dtype": "int64",
         "stats": {
          "unique_count": 2,
          "nan_count": 0,
          "min": "0",
          "max": "1",
          "histogram": [
           {
            "bin_start": 0,
            "bin_end": 0.1,
            "count": 2
           },
           {
            "bin_start": 0.1,
            "bin_end": 0.2,
            "count": 0
           },
           {
            "bin_start": 0.2,
            "bin_end": 0.30000000000000004,
            "count": 0
           },
           {
            "bin_start": 0.30000000000000004,
            "bin_end": 0.4,
            "count": 0
           },
           {
            "bin_start": 0.4,
            "bin_end": 0.5,
            "count": 0
           },
           {
            "bin_start": 0.5,
            "bin_end": 0.6000000000000001,
            "count": 0
           },
           {
            "bin_start": 0.6000000000000001,
            "bin_end": 0.7000000000000001,
            "count": 0
           },
           {
            "bin_start": 0.7000000000000001,
            "bin_end": 0.8,
            "count": 0
           },
           {
            "bin_start": 0.8,
            "bin_end": 0.9,
            "count": 0
           },
           {
            "bin_start": 0.9,
            "bin_end": 1,
            "count": 3
           }
          ]
         }
        },
        {
         "name": "speechiness",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.0327",
          "max": "0.242",
          "histogram": [
           {
            "bin_start": 0.0327,
            "bin_end": 0.05363,
            "count": 2
           },
           {
            "bin_start": 0.05363,
            "bin_end": 0.07455999999999999,
            "count": 0
           },
           {
            "bin_start": 0.07455999999999999,
            "bin_end": 0.09548999999999999,
            "count": 0
           },
           {
            "bin_start": 0.09548999999999999,
            "bin_end": 0.11642,
            "count": 1
           },
           {
            "bin_start": 0.11642,
            "bin_end": 0.13735,
            "count": 0
           },
           {
            "bin_start": 0.13735,
            "bin_end": 0.15827999999999998,
            "count": 0
           },
           {
            "bin_start": 0.15827999999999998,
            "bin_end": 0.17920999999999998,
            "count": 1
           },
           {
            "bin_start": 0.17920999999999998,
            "bin_end": 0.20013999999999998,
            "count": 0
           },
           {
            "bin_start": 0.20013999999999998,
            "bin_end": 0.22107,
            "count": 0
           },
           {
            "bin_start": 0.22107,
            "bin_end": 0.242,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "acousticness",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.658",
          "max": "0.94",
          "histogram": [
           {
            "bin_start": 0.658,
            "bin_end": 0.6862,
            "count": 2
           },
           {
            "bin_start": 0.6862,
            "bin_end": 0.7144,
            "count": 0
           },
           {
            "bin_start": 0.7144,
            "bin_end": 0.7426,
            "count": 0
           },
           {
            "bin_start": 0.7426,
            "bin_end": 0.7708,
            "count": 0
           },
           {
            "bin_start": 0.7708,
            "bin_end": 0.7989999999999999,
            "count": 0
           },
           {
            "bin_start": 0.7989999999999999,
            "bin_end": 0.8271999999999999,
            "count": 1
           },
           {
            "bin_start": 0.8271999999999999,
            "bin_end": 0.8553999999999999,
            "count": 0
           },
           {
            "bin_start": 0.8553999999999999,
            "bin_end": 0.8835999999999999,
            "count": 0
           },
           {
            "bin_start": 0.8835999999999999,
            "bin_end": 0.9117999999999999,
            "count": 0
           },
           {
            "bin_start": 0.9117999999999999,
            "bin_end": 0.94,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "instrumentalness",
         "dtype": "float64",
         "stats": {
          "unique_count": 4,
          "nan_count": 0,
          "min": "0.0",
          "max": "0.922",
          "histogram": [
           {
            "bin_start": 0,
            "bin_end": 0.0922,
            "count": 3
           },
           {
            "bin_start": 0.0922,
            "bin_end": 0.1844,
            "count": 0
           },
           {
            "bin_start": 0.1844,
            "bin_end": 0.2766,
            "count": 0
           },
           {
            "bin_start": 0.2766,
            "bin_end": 0.3688,
            "count": 0
           },
           {
            "bin_start": 0.3688,
            "bin_end": 0.461,
            "count": 0
           },
           {
            "bin_start": 0.461,
            "bin_end": 0.5532,
            "count": 0
           },
           {
            "bin_start": 0.5532,
            "bin_end": 0.6454,
            "count": 0
           },
           {
            "bin_start": 0.6454,
            "bin_end": 0.7376,
            "count": 0
           },
           {
            "bin_start": 0.7376,
            "bin_end": 0.8298000000000001,
            "count": 0
           },
           {
            "bin_start": 0.8298000000000001,
            "bin_end": 0.922,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "liveness",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.107",
          "max": "0.738",
          "histogram": [
           {
            "bin_start": 0.107,
            "bin_end": 0.1701,
            "count": 3
           },
           {
            "bin_start": 0.1701,
            "bin_end": 0.23320000000000002,
            "count": 0
           },
           {
            "bin_start": 0.23320000000000002,
            "bin_end": 0.2963,
            "count": 0
           },
           {
            "bin_start": 0.2963,
            "bin_end": 0.3594,
            "count": 0
           },
           {
            "bin_start": 0.3594,
            "bin_end": 0.4225,
            "count": 1
           },
           {
            "bin_start": 0.4225,
            "bin_end": 0.48560000000000003,
            "count": 0
           },
           {
            "bin_start": 0.48560000000000003,
            "bin_end": 0.5487000000000001,
            "count": 0
           },
           {
            "bin_start": 0.5487000000000001,
            "bin_end": 0.6118,
            "count": 0
           },
           {
            "bin_start": 0.6118,
            "bin_end": 0.6749,
            "count": 0
           },
           {
            "bin_start": 0.6749,
            "bin_end": 0.738,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "valence",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.139",
          "max": "0.853",
          "histogram": [
           {
            "bin_start": 0.139,
            "bin_end": 0.2104,
            "count": 2
           },
           {
            "bin_start": 0.2104,
            "bin_end": 0.2818,
            "count": 0
           },
           {
            "bin_start": 0.2818,
            "bin_end": 0.35319999999999996,
            "count": 0
           },
           {
            "bin_start": 0.35319999999999996,
            "bin_end": 0.4246,
            "count": 0
           },
           {
            "bin_start": 0.4246,
            "bin_end": 0.496,
            "count": 1
           },
           {
            "bin_start": 0.496,
            "bin_end": 0.5673999999999999,
            "count": 0
           },
           {
            "bin_start": 0.5673999999999999,
            "bin_end": 0.6387999999999999,
            "count": 0
           },
           {
            "bin_start": 0.6387999999999999,
            "bin_end": 0.7101999999999999,
            "count": 0
           },
           {
            "bin_start": 0.7101999999999999,
            "bin_end": 0.7816,
            "count": 0
           },
           {
            "bin_start": 0.7816,
            "bin_end": 0.853,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "tempo",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "92.472",
          "max": "137.976",
          "histogram": [
           {
            "bin_start": 92.472,
            "bin_end": 97.02239999999999,
            "count": 1
           },
           {
            "bin_start": 97.02239999999999,
            "bin_end": 101.5728,
            "count": 0
           },
           {
            "bin_start": 101.5728,
            "bin_end": 106.1232,
            "count": 0
           },
           {
            "bin_start": 106.1232,
            "bin_end": 110.6736,
            "count": 0
           },
           {
            "bin_start": 110.6736,
            "bin_end": 115.22399999999999,
            "count": 0
           },
           {
            "bin_start": 115.22399999999999,
            "bin_end": 119.7744,
            "count": 0
           },
           {
            "bin_start": 119.7744,
            "bin_end": 124.3248,
            "count": 1
           },
           {
            "bin_start": 124.3248,
            "bin_end": 128.8752,
            "count": 0
           },
           {
            "bin_start": 128.8752,
            "bin_end": 133.4256,
            "count": 1
           },
           {
            "bin_start": 133.4256,
            "bin_end": 137.976,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "duration_ms",
         "dtype": "int64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "26244",
          "max": "164227",
          "histogram": [
           {
            "bin_start": 26244,
            "bin_end": 40042.3,
            "count": 1
           },
           {
            "bin_start": 40042.3,
            "bin_end": 53840.6,
            "count": 0
           },
           {
            "bin_start": 53840.6,
            "bin_end": 67638.9,
            "count": 0
           },
           {
            "bin_start": 67638.9,
            "bin_end": 81437.2,
            "count": 0
           },
           {
            "bin_start": 81437.2,
            "bin_end": 95235.5,
            "count": 0
           },
           {
            "bin_start": 95235.5,
            "bin_end": 109033.79999999999,
            "count": 0
           },
           {
            "bin_start": 109033.79999999999,
            "bin_end": 122832.09999999999,
            "count": 0
           },
           {
            "bin_start": 122832.09999999999,
            "bin_end": 136630.4,
            "count": 0
           },
           {
            "bin_start": 136630.4,
            "bin_end": 150428.7,
            "count": 2
           },
           {
            "bin_start": 150428.7,
            "bin_end": 164227,
            "count": 2
           }
          ]
         }
        },
        {
         "name": "time_signature",
         "dtype": "int64",
         "stats": {
          "unique_count": 2,
          "nan_count": 0,
          "min": "3",
          "max": "4",
          "histogram": [
           {
            "bin_start": 3,
            "bin_end": 3.1,
            "count": 1
           },
           {
            "bin_start": 3.1,
            "bin_end": 3.2,
            "count": 0
           },
           {
            "bin_start": 3.2,
            "bin_end": 3.3,
            "count": 0
           },
           {
            "bin_start": 3.3,
            "bin_end": 3.4,
            "count": 0
           },
           {
            "bin_start": 3.4,
            "bin_end": 3.5,
            "count": 0
           },
           {
            "bin_start": 3.5,
            "bin_end": 3.6,
            "count": 0
           },
           {
            "bin_start": 3.6,
            "bin_end": 3.7,
            "count": 0
           },
           {
            "bin_start": 3.7,
            "bin_end": 3.8,
            "count": 0
           },
           {
            "bin_start": 3.8,
            "bin_end": 3.9,
            "count": 0
           },
           {
            "bin_start": 3.9,
            "bin_end": 4,
            "count": 4
           }
          ]
         }
        },
        {
         "name": "chorus_hit",
         "dtype": "float64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "0.0",
          "max": "56.63498",
          "histogram": [
           {
            "bin_start": 0,
            "bin_end": 5.663498,
            "count": 1
           },
           {
            "bin_start": 5.663498,
            "bin_end": 11.326996,
            "count": 0
           },
           {
            "bin_start": 11.326996,
            "bin_end": 16.990493999999998,
            "count": 0
           },
           {
            "bin_start": 16.990493999999998,
            "bin_end": 22.653992,
            "count": 0
           },
           {
            "bin_start": 22.653992,
            "bin_end": 28.31749,
            "count": 1
           },
           {
            "bin_start": 28.31749,
            "bin_end": 33.980987999999996,
            "count": 1
           },
           {
            "bin_start": 33.980987999999996,
            "bin_end": 39.644486,
            "count": 1
           },
           {
            "bin_start": 39.644486,
            "bin_end": 45.307984,
            "count": 0
           },
           {
            "bin_start": 45.307984,
            "bin_end": 50.971481999999995,
            "count": 0
           },
           {
            "bin_start": 50.971481999999995,
            "bin_end": 56.63498,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "sections",
         "dtype": "int64",
         "stats": {
          "unique_count": 5,
          "nan_count": 0,
          "min": "2",
          "max": "10",
          "histogram": [
           {
            "bin_start": 2,
            "bin_end": 2.8,
            "count": 1
           },
           {
            "bin_start": 2.8,
            "bin_end": 3.6,
            "count": 0
           },
           {
            "bin_start": 3.6,
            "bin_end": 4.4,
            "count": 0
           },
           {
            "bin_start": 4.4,
            "bin_end": 5.2,
            "count": 0
           },
           {
            "bin_start": 5.2,
            "bin_end": 6,
            "count": 0
           },
           {
            "bin_start": 6,
            "bin_end": 6.800000000000001,
            "count": 1
           },
           {
            "bin_start": 6.800000000000001,
            "bin_end": 7.6000000000000005,
            "count": 0
           },
           {
            "bin_start": 7.6000000000000005,
            "bin_end": 8.4,
            "count": 1
           },
           {
            "bin_start": 8.4,
            "bin_end": 9.2,
            "count": 1
           },
           {
            "bin_start": 9.2,
            "bin_end": 10,
            "count": 1
           }
          ]
         }
        },
        {
         "name": "target",
         "dtype": "int64",
         "stats": {
          "unique_count": 1,
          "nan_count": 0,
          "min": "0",
          "max": "0",
          "histogram": [
           {
            "bin_start": -0.5,
            "bin_end": -0.4,
            "count": 0
           },
           {
            "bin_start": -0.4,
            "bin_end": -0.3,
            "count": 0
           },
           {
            "bin_start": -0.3,
            "bin_end": -0.19999999999999996,
            "count": 0
           },
           {
            "bin_start": -0.19999999999999996,
            "bin_end": -0.09999999999999998,
            "count": 0
           },
           {
            "bin_start": -0.09999999999999998,
            "bin_end": 0,
            "count": 0
           },
           {
            "bin_start": 0,
            "bin_end": 0.10000000000000009,
            "count": 5
           },
           {
            "bin_start": 0.10000000000000009,
            "bin_end": 0.20000000000000007,
            "count": 0
           },
           {
            "bin_start": 0.20000000000000007,
            "bin_end": 0.30000000000000004,
            "count": 0
           },
           {
            "bin_start": 0.30000000000000004,
            "bin_end": 0.4,
            "count": 0
           },
           {
            "bin_start": 0.4,
            "bin_end": 0.5,
            "count": 0
           }
          ]
         }
        },
        {
         "name": "decade",
         "dtype": "int64",
         "stats": {
          "unique_count": 1,
          "nan_count": 0,
          "min": "1960",
          "max": "1960",
          "histogram": [
           {
            "bin_start": 1959.5,
            "bin_end": 1959.6,
            "count": 0
           },
           {
            "bin_start": 1959.6,
            "bin_end": 1959.7,
            "count": 0
           },
           {
            "bin_start": 1959.7,
            "bin_end": 1959.8,
            "count": 0
           },
           {
            "bin_start": 1959.8,
            "bin_end": 1959.9,
            "count": 0
           },
           {
            "bin_start": 1959.9,
            "bin_end": 1960,
            "count": 0
           },
           {
            "bin_start": 1960,
            "bin_end": 1960.1,
            "count": 5
           },
           {
            "bin_start": 1960.1,
            "bin_end": 1960.2,
            "count": 0
           },
           {
            "bin_start": 1960.2,
            "bin_end": 1960.3,
            "count": 0
           },
           {
            "bin_start": 1960.3,
            "bin_end": 1960.4,
            "count": 0
           },
           {
            "bin_start": 1960.4,
            "bin_end": 1960.5,
            "count": 0
           }
          ]
         }
        },
        {
         "name": "_deepnote_index_column",
         "dtype": "int64"
        }
       ],
       "rows_top": [
        {
         "track": "Le ire di Ursus",
         "artist": "Giampiero Boneschi",
         "uri": "spotify:track:3dVgolCVuNgecU5jovpc45",
         "danceability": 0.42,
         "energy": 0.873,
         "key": 3,
         "loudness": -11.689,
         "mode": 1,
         "speechiness": 0.1,
         "acousticness": 0.808,
         "instrumentalness": 0.879,
         "liveness": 0.363,
         "valence": 0.183,
         "tempo": 119.821,
         "duration_ms": 26244,
         "time_signature": 4,
         "chorus_hit": 0,
         "sections": 2,
         "target": 0,
         "decade": 1960,
         "_deepnote_index_column": 41101
        },
        {
         "track": "A-B-C",
         "artist": "John Zacherle",
         "uri": "spotify:track:0zXhzKf4raEYstebL7L42C",
         "danceability": 0.716,
         "energy": 0.742,
         "key": 10,
         "loudness": -5.727,
         "mode": 0,
         "speechiness": 0.242,
         "acousticness": 0.663,
         "instrumentalness": 0,
         "liveness": 0.107,
         "valence": 0.834,
         "tempo": 135.104,
         "duration_ms": 142987,
         "time_signature": 4,
         "chorus_hit": 28.34316,
         "sections": 10,
         "target": 0,
         "decade": 1960,
         "_deepnote_index_column": 41102
        },
        {
         "track": "Useless Landscape",
         "artist": "Antônio Carlos Jobim",
         "uri": "spotify:track:2uC02PCRVM6BhAaK7dgxxp",
         "danceability": 0.361,
         "energy": 0.0944,
         "key": 10,
         "loudness": -22.663,
         "mode": 1,
         "speechiness": 0.0327,
         "acousticness": 0.94,
         "instrumentalness": 0.000178,
         "liveness": 0.115,
         "valence": 0.139,
         "tempo": 92.472,
         "duration_ms": 137200,
         "time_signature": 4,
         "chorus_hit": 56.63498,
         "sections": 6,
         "target": 0,
         "decade": 1960,
         "_deepnote_index_column": 41103
        },
        {
         "track": "Vendedor De Assaí",
         "artist": "Ary Lobo",
         "uri": "spotify:track:0gpSUWZWNIRrEenaC1ZGk9",
         "danceability": 0.452,
         "energy": 0.759,
         "key": 4,
         "loudness": -7.821,
         "mode": 0,
         "speechiness": 0.175,
         "acousticness": 0.658,
         "instrumentalness": 0,
         "liveness": 0.738,
         "valence": 0.853,
         "tempo": 137.976,
         "duration_ms": 159107,
         "time_signature": 3,
         "chorus_hit": 28.24372,
         "sections": 9,
         "target": 0,
         "decade": 1960,
         "_deepnote_index_column": 41104
        },
        {
         "track": "Mambo Simphony - Remasterizado",
         "artist": "Orquesta Casino De La Playa",
         "uri": "spotify:track:5j0G0nkCSPlmSCSgXchMn3",
         "danceability": 0.554,
         "energy": 0.278,
         "key": 5,
         "loudness": -8.386,
         "mode": 1,
         "speechiness": 0.0349,
         "acousticness": 0.928,
         "instrumentalness": 0.922,
         "liveness": 0.136,
         "valence": 0.428,
         "tempo": 131.401,
         "duration_ms": 164227,
         "time_signature": 4,
         "chorus_hit": 37.39251,
         "sections": 8,
         "target": 0,
         "decade": 1960,
         "_deepnote_index_column": 41105
        }
       ],
       "rows_bottom": null
      },
      "text/plain": "                                track                       artist  \\\n41101                 Le ire di Ursus           Giampiero Boneschi   \n41102                           A-B-C                John Zacherle   \n41103               Useless Landscape         Antônio Carlos Jobim   \n41104               Vendedor De Assaí                     Ary Lobo   \n41105  Mambo Simphony - Remasterizado  Orquesta Casino De La Playa   \n\n                                        uri  danceability  energy  key  \\\n41101  spotify:track:3dVgolCVuNgecU5jovpc45         0.420  0.8730    3   \n41102  spotify:track:0zXhzKf4raEYstebL7L42C         0.716  0.7420   10   \n41103  spotify:track:2uC02PCRVM6BhAaK7dgxxp         0.361  0.0944   10   \n41104  spotify:track:0gpSUWZWNIRrEenaC1ZGk9         0.452  0.7590    4   \n41105  spotify:track:5j0G0nkCSPlmSCSgXchMn3         0.554  0.2780    5   \n\n       loudness  mode  speechiness  acousticness  instrumentalness  liveness  \\\n41101   -11.689     1       0.1000         0.808          0.879000     0.363   \n41102    -5.727     0       0.2420         0.663          0.000000     0.107   \n41103   -22.663     1       0.0327         0.940          0.000178     0.115   \n41104    -7.821     0       0.1750         0.658          0.000000     0.738   \n41105    -8.386     1       0.0349         0.928          0.922000     0.136   \n\n       valence    tempo  duration_ms  time_signature  chorus_hit  sections  \\\n41101    0.183  119.821        26244               4     0.00000         2   \n41102    0.834  135.104       142987               4    28.34316        10   \n41103    0.139   92.472       137200               4    56.63498         6   \n41104    0.853  137.976       159107               3    28.24372         9   \n41105    0.428  131.401       164227               4    37.39251         8   \n\n       target  decade  \n41101       0    1960  \n41102       0    1960  \n41103       0    1960  \n41104       0    1960  \n41105       0    1960  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>track</th>\n      <th>artist</th>\n      <th>uri</th>\n      <th>danceability</th>\n      <th>energy</th>\n      <th>key</th>\n      <th>loudness</th>\n      <th>mode</th>\n      <th>speechiness</th>\n      <th>acousticness</th>\n      <th>instrumentalness</th>\n      <th>liveness</th>\n      <th>valence</th>\n      <th>tempo</th>\n      <th>duration_ms</th>\n      <th>time_signature</th>\n      <th>chorus_hit</th>\n      <th>sections</th>\n      <th>target</th>\n      <th>decade</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>41101</th>\n      <td>Le ire di Ursus</td>\n      <td>Giampiero Boneschi</td>\n      <td>spotify:track:3dVgolCVuNgecU5jovpc45</td>\n      <td>0.420</td>\n      <td>0.8730</td>\n      <td>3</td>\n      <td>-11.689</td>\n      <td>1</td>\n      <td>0.1000</td>\n      <td>0.808</td>\n      <td>0.879000</td>\n      <td>0.363</td>\n      <td>0.183</td>\n      <td>119.821</td>\n      <td>26244</td>\n      <td>4</td>\n      <td>0.00000</td>\n      <td>2</td>\n      <td>0</td>\n      <td>1960</td>\n    </tr>\n    <tr>\n      <th>41102</th>\n      <td>A-B-C</td>\n      <td>John Zacherle</td>\n      <td>spotify:track:0zXhzKf4raEYstebL7L42C</td>\n      <td>0.716</td>\n      <td>0.7420</td>\n      <td>10</td>\n      <td>-5.727</td>\n      <td>0</td>\n      <td>0.2420</td>\n      <td>0.663</td>\n      <td>0.000000</td>\n      <td>0.107</td>\n      <td>0.834</td>\n      <td>135.104</td>\n      <td>142987</td>\n      <td>4</td>\n      <td>28.34316</td>\n      <td>10</td>\n      <td>0</td>\n      <td>1960</td>\n    </tr>\n    <tr>\n      <th>41103</th>\n      <td>Useless Landscape</td>\n      <td>Antônio Carlos Jobim</td>\n      <td>spotify:track:2uC02PCRVM6BhAaK7dgxxp</td>\n      <td>0.361</td>\n      <td>0.0944</td>\n      <td>10</td>\n      <td>-22.663</td>\n      <td>1</td>\n      <td>0.0327</td>\n      <td>0.940</td>\n      <td>0.000178</td>\n      <td>0.115</td>\n      <td>0.139</td>\n      <td>92.472</td>\n      <td>137200</td>\n      <td>4</td>\n      <td>56.63498</td>\n      <td>6</td>\n      <td>0</td>\n      <td>1960</td>\n    </tr>\n    <tr>\n      <th>41104</th>\n      <td>Vendedor De Assaí</td>\n      <td>Ary Lobo</td>\n      <td>spotify:track:0gpSUWZWNIRrEenaC1ZGk9</td>\n      <td>0.452</td>\n      <td>0.7590</td>\n      <td>4</td>\n      <td>-7.821</td>\n      <td>0</td>\n      <td>0.1750</td>\n      <td>0.658</td>\n      <td>0.000000</td>\n      <td>0.738</td>\n      <td>0.853</td>\n      <td>137.976</td>\n      <td>159107</td>\n      <td>3</td>\n      <td>28.24372</td>\n      <td>9</td>\n      <td>0</td>\n      <td>1960</td>\n    </tr>\n    <tr>\n      <th>41105</th>\n      <td>Mambo Simphony - Remasterizado</td>\n      <td>Orquesta Casino De La Playa</td>\n      <td>spotify:track:5j0G0nkCSPlmSCSgXchMn3</td>\n      <td>0.554</td>\n      <td>0.2780</td>\n      <td>5</td>\n      <td>-8.386</td>\n      <td>1</td>\n      <td>0.0349</td>\n      <td>0.928</td>\n      <td>0.922000</td>\n      <td>0.136</td>\n      <td>0.428</td>\n      <td>131.401</td>\n      <td>164227</td>\n      <td>4</td>\n      <td>37.39251</td>\n      <td>8</td>\n      <td>0</td>\n      <td>1960</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "29a140c6",
    "execution_start": 1638642121392,
    "execution_millis": 35,
    "is_code_hidden": false,
    "is_output_hidden": false,
    "cell_id": "00007-2f9732aa-32e6-48e6-b65c-1238a78e8a04",
    "deepnote_output_heights": [
     232.140625
    ],
    "deepnote_cell_type": "code"
   },
   "source": "# Add custom columns that will be useful later\n\n# Duration of song in seconds\ndata[\"duration_sec\"] = np.round(data[\"duration_ms\"]*(1/1000))\ndata[\"duration_sec\"]\n\n# Loudness of sound (rounded)\ndata[\"loudness_rounded\"] = np.round(data[\"loudness\"])\ndata[\"loudness_rounded\"]",
   "execution_count": null,
   "outputs": [
    {
     "output_type": "execute_result",
     "execution_count": 5,
     "data": {
      "text/plain": "0        -5.0\n1       -15.0\n2        -7.0\n3        -6.0\n4        -4.0\n         ... \n41101   -12.0\n41102    -6.0\n41103   -23.0\n41104    -8.0\n41105    -8.0\nName: loudness_rounded, Length: 41106, dtype: float64"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00007-73833ebf-bb11-44fd-92b9-b51c68e9bf9b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "a34b193c",
    "execution_start": 1638642353951,
    "execution_millis": 3715,
    "deepnote_cell_type": "code"
   },
   "source": "# For later analysis of genres. Below data is from spotify playlists. \ndata.to_csv(\"./data.csv\")\n\n#For a clean copy for genre analysis\ncData = data.copy()\nrData = data.copy()\n\n# Load the Genre Songs from the csv into respective dataframes\nchristmas_songs = pd.read_csv('./datasets/christmas_hits.csv')\ncountry_songs = pd.read_csv('./datasets/country_hits.csv')\nrb_songs = pd.read_csv('./datasets/rb_hits.csv')\nmetal_songs = pd.read_csv('./datasets/metal_hits.csv')\nhiphop_songs = pd.read_csv('./datasets/hiphop_hits.csv')\n\n# Add a boolean value column indicating whether the song is in the playlist \"Christmas Hits\" by Spotify (https://open.spotify.com/playlist/37i9dQZF1DX0Yxoavh5qJV?si=96f9678a2982444d)\ndata['christmas'] = data.apply(lambda row: True if (row[\"track\"] in christmas_songs[\"Track name\"].tolist()) and (row[\"artist\"] in christmas_songs[\" Artist name\"].tolist()) else False, axis=1)\ndata['country'] = data.apply(lambda row: True if (row[\"track\"] in country_songs[\"Track name\"].tolist()) and (row[\"artist\"] in country_songs[\" Artist name\"].tolist()) else False, axis=1)\ndata['rb'] = data.apply(lambda row: True if (row[\"track\"] in rb_songs[\"Track name\"].tolist()) and (row[\"artist\"] in rb_songs[\" Artist name\"].tolist()) else False, axis=1)\ndata['metal'] = data.apply(lambda row: True if (row[\"track\"] in metal_songs[\"Track name\"].tolist()) and (row[\"artist\"] in metal_songs[\" Artist name\"].tolist()) else False, axis=1)\ndata['hiphop'] = data.apply(lambda row: True if (row[\"track\"] in hiphop_songs[\"Track name\"].tolist()) and (row[\"artist\"] in hiphop_songs[\" Artist name\"].tolist()) else False, axis=1)\n\nprint(\"Christmas: \", len(data.loc[data['christmas'] == True]))\nprint(\"Country: \", len(data.loc[data['country'] == True]))\nprint(\"RB: \", len(data.loc[data['rb'] == True]))\nprint(\"Metal: \", len(data.loc[data['metal'] == True]))\nprint(\"Hip-Hop: \", len(data.loc[data['hiphop'] == True]))\n",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "Christmas:  22\nCountry:  27\nRB:  10\nMetal:  9\nHip-Hop:  8\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "#### General statistics\nMeans, medians, modes, standard deviations of each column for both unique decades and the overall dataset",
   "metadata": {
    "tags": [],
    "cell_id": "00009-3486b236-10dd-4f65-b2a1-5e3443e00909",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "markdown",
   "source": "#### Cumulative statistical analysis\nAnalysis of relationships between each column and 'hits' vs 'flops' for all decades overall\n",
   "metadata": {
    "tags": [],
    "cell_id": "00009-678c65f6-b8a4-4221-9d84-f837267c89f5",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "markdown",
   "source": "#### Decade-related statistical analysis\n\nAnalysis of properties of songs and how they vary by decade",
   "metadata": {
    "tags": [],
    "cell_id": "00012-4d8c5959-387b-4943-b253-0745a38695c3",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "markdown",
   "source": "#### Genre-Specific Hits statistical analysis\nAnalysis of relationships between properties of the hits songs between different genres/categories of songs. Includes hypothesis tests and bootstrapping\n",
   "metadata": {
    "tags": [],
    "cell_id": "00010-72881f27-4ec7-4d37-be8e-35137984ac38",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00012-f414a8cb-46cc-4236-92fb-a5a4583d4e6f",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "c757cc0f",
    "execution_start": 1638642125030,
    "execution_millis": 21,
    "deepnote_cell_type": "code"
   },
   "source": "# Analysis of proportion of mode=1 for christmas hits and non-christmas hits\n# Note that a mode of 1 implies \"happy\" songs as they are on the major scale. \n# Note that a mode of 0 implies \"sad\" songs as they are on the minor scale\n\n# Null hypothesis: True Proportion of happy in Christmas = True Proportion of happy in non-christmas\n# Alternative hypothesis: True Proportion of happy in Christmas > True Proportion of happy in non-christmas\n\nmask_christmas = data[\"christmas\"] == True\nmask_non_christmas = data[\"christmas\"] == False\n\ncommon_christmas_songs = data[mask_christmas]\ncommon_non_christmas_songs = data[mask_non_christmas]\n\nchristmas_hits = common_christmas_songs[common_christmas_songs[\"target\"] == 1]\nnon_christmas_hits =common_non_christmas_songs[common_non_christmas_songs[\"target\"] == 1]\n\nmask_christmas_mode = christmas_hits[\"mode\"] == 1\nmask_non_christmas_mode = non_christmas_hits[\"mode\"] == 1\n\nproportion_happy_christmas = len(christmas_hits[mask_christmas_mode])/len(christmas_hits[\"mode\"])\n\nnum_bootstrap = 10000\nbootstrap_samples = np.random.choice(non_christmas_hits[\"mode\"], size=(num_bootstrap, len(christmas_hits[\"mode\"])), replace=True)\nsampling_proportions_non_christmas = np.average(bootstrap_samples, axis=1)\nnon_christmas_proportion_avg = np.average(sampling_proportions_non_christmas)\n\nsd_christmas = np.sqrt(proportion_happy_christmas*(1-proportion_happy_christmas))\nsd_non_christmas = np.sqrt(non_christmas_proportion_avg*(1-non_christmas_proportion_avg))\n\nnewpval_christmas = stats.ttest_ind_from_stats(mean1=proportion_happy_christmas, std1=sd_christmas, nobs1=len(christmas_hits[\"mode\"]), mean2=non_christmas_proportion_avg, std2=sd_non_christmas, nobs2=len(christmas_hits[\"mode\"]))\np_value_bootstrapping_christmas = newpval_christmas[1]/2\nprint(p_value_bootstrapping_christmas)\n\n# Note that p value is approx. 0.112, which is greater than the alpha level of 0.05, so we fail to reject the null hypothesis. ",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "0.1126719975553776\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00013-e5cbcbd3-2a70-4e50-8099-a8f84ee8f837",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "4ddfce37",
    "execution_start": 1638642125057,
    "execution_millis": 26,
    "deepnote_cell_type": "code"
   },
   "source": "# Analysis of the Instrumentalness of country songs\n# Null Hypothesis: Mean instrumentalness of Country = Mean instrumentalness of non-country\n# Alternative Hypothesis: Mean instrumentalness of Country > Mean instrumentalness of non-country\n\nmask_country = data[\"country\"] == True\nmask_non_country = data[\"country\"] == False\n\ncommon_country_songs = data[mask_country]\ncommon_non_country_songs = data[mask_non_country]\n\ncountry_hits = common_country_songs[common_country_songs[\"target\"]==1]\nnon_country_hits = common_non_country_songs[common_non_country_songs[\"target\"]==1]\n\nmean_instrumental_country = np.average(country_hits[\"instrumentalness\"])\n\nnum_bootstrap = 10000\nbootstrap_samples_country = np.random.choice(non_country_hits[\"instrumentalness\"], size=(num_bootstrap, len(country_hits[\"instrumentalness\"])), replace=True)\nsampling_proportions_non_country = np.average(bootstrap_samples_country, axis=1)\nmean_instrumental_non_country = np.average(sampling_proportions_non_country)\n\ncountry_sd = np.std(country_hits[\"instrumentalness\"])\nnon_country_sd = np.std(non_country_hits[\"instrumentalness\"])\n\nnewpval_country = stats.ttest_ind_from_stats(mean1=mean_instrumental_country, std1=country_sd, nobs1=len(country_hits[\"instrumentalness\"]), mean2=mean_instrumental_non_country, std2=non_country_sd, nobs2=len(country_hits[\"instrumentalness\"]))\np_value_bootstrapping_country = newpval_country[1]/2\nprint(p_value_bootstrapping_country)\n\n# Note that p value is approx. 0.114, which is greater than the alpha level of 0.05, so we fail to reject the null hypothesis. ",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "0.10911082974509444\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00013-3b0fd3ff-13a2-4e58-8091-af1adeb0544b",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "370ca329",
    "execution_start": 1638642125139,
    "execution_millis": 438,
    "deepnote_cell_type": "code"
   },
   "source": "# Analysis of the tempo of Instrumental (and non-instrumental) songs\n# Two-tailed test\n# Null Hypothesis: Mean tempo of instrumental songs = Mean tempo of non instrumental songs\n# Alternative Hypothesis: Mean tempo of instrumental songs not equal to mean tempo of non-instrumental songs\n\n# A song \"isInstrumental\" if the instrumentalness is greater than 0.5\ndata['isInstrumental'] = data.apply(lambda row: True if (row[\"instrumentalness\"]>0.5) else False, axis=1)\nmask_instrumental = data[\"isInstrumental\"] == True\nmask_non_instrumental = data[\"isInstrumental\"] == False\n\ncommon_instrumental_songs = data[mask_instrumental]\ncommon_non_instrumental_songs = data[mask_non_instrumental]\n\ninstrumental_hits = common_instrumental_songs[common_instrumental_songs[\"target\"]==1]\nnon_instrumental_hits = common_non_instrumental_songs[common_non_instrumental_songs[\"target\"]==1]\n\nmean_tempo_instrumental = np.average(instrumental_hits[\"tempo\"])\n\nnum_bootstrap = 10000\nbootstrap_samples_instrumental = np.random.choice(non_instrumental_hits[\"tempo\"], size=(num_bootstrap, len(instrumental_hits[\"tempo\"])), replace=True)\nsampling_proportions_non_instrumental = np.average(bootstrap_samples_instrumental, axis=1)\nmean_tempo_non_instrumental = np.average(sampling_proportions_non_instrumental)\n\ninstrumental_sd = np.std(instrumental_hits[\"tempo\"])\nnon_instrumental_sd = np.std(non_instrumental_hits[\"tempo\"])\n\nnewpval_instrumental = stats.ttest_ind_from_stats(mean1=mean_tempo_instrumental, std1=instrumental_sd, nobs1=len(instrumental_hits[\"tempo\"]), mean2=mean_tempo_non_instrumental, std2=non_instrumental_sd, nobs2=len(instrumental_hits[\"tempo\"]))\np_value_bootstrapping_instrumental = newpval_instrumental[1] \nprint(p_value_bootstrapping_instrumental)",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "text": "0.5165217722121881\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "#### One-hit wonders and Consistent Hitters statistical analysis\n\nAnalysis of properties of songs by artists with one-hit wonders vs. artists with multiple hit songs",
   "metadata": {
    "tags": [],
    "cell_id": "00011-7a0f314b-b4d8-4c95-b4e7-e2d1b7fcf488",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "markdown",
   "source": "#### Regression statistical analysis",
   "metadata": {
    "tags": [],
    "cell_id": "00013-3b28056a-ae84-4008-b069-be65eceea528",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "markdown",
   "source": "#### Backwards regression statistical analysis",
   "metadata": {
    "tags": [],
    "cell_id": "00017-6e3b4270-e9c2-4462-b9b7-bd04e8fda657",
    "deepnote_cell_type": "markdown"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00014-72728462-7423-44d6-aa45-8a3772663b93",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "3c27c8d6",
    "execution_start": 1638642125578,
    "execution_millis": 2,
    "deepnote_cell_type": "code"
   },
   "source": "#### Backwards regression statistical analysis",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "<a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=162d76af-48a2-4f75-9aef-34643a48aa39' target=\"_blank\">\n<img alt='Created in deepnote.com' style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ2h0PSI4MHB4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPgogICAgICAgIDxnIGlkPSJBcnRib2FyZCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTEyMzUuMDAwMDAwLCAtNzkuMDAwMDAwKSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD0iUGF0aC0yMCIgZmlsbD0iIzAyNjVCNCIgcG9pbnRzPSIyLjM3NjIzNzYyIDgwIDM4LjA0NzY2NjcgODAgNTcuODIxNzgyMiA3My44MDU3NTkyIDU3LjgyMTc4MjIgMzIuNzU5MjczOSAzOS4xNDAyMjc4IDMxLjY4MzE2ODMiPjwvcG9seWdvbj4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0zNS4wMDc3MTgsODAgQzQyLjkwNjIwMDcsNzYuNDU0OTM1OCA0Ny41NjQ5MTY3LDcxLjU0MjI2NzEgNDguOTgzODY2LDY1LjI2MTk5MzkgQzUxLjExMjI4OTksNTUuODQxNTg0MiA0MS42NzcxNzk1LDQ5LjIxMjIyODQgMjUuNjIzOTg0Niw0OS4yMTIyMjg0IEMyNS40ODQ5Mjg5LDQ5LjEyNjg0NDggMjkuODI2MTI5Niw0My4yODM4MjQ4IDM4LjY0NzU4NjksMzEuNjgzMTY4MyBMNzIuODcxMjg3MSwzMi41NTQ0MjUgTDY1LjI4MDk3Myw2Ny42NzYzNDIxIEw1MS4xMTIyODk5LDc3LjM3NjE0NCBMMzUuMDA3NzE4LDgwIFoiIGlkPSJQYXRoLTIyIiBmaWxsPSIjMDAyODY4Ij48L3BhdGg+CiAgICAgICAgICAgICAgICA8cGF0aCBkPSJNMCwzNy43MzA0NDA1IEwyNy4xMTQ1MzcsMC4yNTcxMTE0MzYgQzYyLjM3MTUxMjMsLTEuOTkwNzE3MDEgODAsMTAuNTAwMzkyNyA4MCwzNy43MzA0NDA1IEM4MCw2NC45NjA0ODgyIDY0Ljc3NjUwMzgsNzkuMDUwMzQxNCAzNC4zMjk1MTEzLDgwIEM0Ny4wNTUzNDg5LDc3LjU2NzA4MDggNTMuNDE4MjY3Nyw3MC4zMTM2MTAzIDUzLjQxODI2NzcsNTguMjM5NTg4NSBDNTMuNDE4MjY3Nyw0MC4xMjg1NTU3IDM2LjMwMzk1NDQsMzcuNzMwNDQwNSAyNS4yMjc0MTcsMzcuNzMwNDQwNSBDMTcuODQzMDU4NiwzNy43MzA0NDA1IDkuNDMzOTE5NjYsMzcuNzMwNDQwNSAwLDM3LjczMDQ0MDUgWiIgaWQ9IlBhdGgtMTkiIGZpbGw9IiMzNzkzRUYiPjwvcGF0aD4KICAgICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+' > </img>\nCreated in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>",
   "metadata": {
    "tags": [],
    "created_in_deepnote_cell": true,
    "deepnote_cell_type": "markdown"
   }
  }
 ],
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "orig_nbformat": 2,
  "deepnote": {
   "is_reactive": false
  },
  "deepnote_notebook_id": "3e42dbdd-b2f1-4d4f-9386-6e889f935c0d",
  "deepnote_execution_queue": []
 }
}