get_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f66ab377",
   "metadata": {},
   "source": [
    "# Query Spotify data on daily/weekly chart playlists\n",
    "\n",
    "2022 June 05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4b89003e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division, print_function\n",
    "\n",
    "#import json\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b724f36a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import coding_club"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f527c658",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for testing with expired tokens\n",
    "#token = \"BQDqHhMHQ8Foyth_kehVIYYmM_GR1_d8aBg3H3xQmxNF77FkOLBtnbBsGZ41JhXYIDcHQAP4qP9Qio7rDdY\"\n",
    "# good until ~1640 on Sunday June 05, 2022\n",
    "token = \"BQC5M9AKsYreUsSMz7C_IhnpczsRU2orTUt1lYOFQSt7RT3OVL9XRWuBWsT0Q6zcGnkhen7aXBZo8z9UjDg\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "987fbb03",
   "metadata": {},
   "source": [
    "# Generic Spotify API query methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d060dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def spquery(url, token, **kwargs):\n",
    "    \"\"\"\n",
    "    HTTPS GET query to Spotify Web API using client credentials,\n",
    "    returns JSON-formatted output\n",
    "    \"\"\"\n",
    "    x = requests.get(\n",
    "        url,\n",
    "        headers = {\n",
    "            \"Content-Type\": \"application/json\",\n",
    "            \"Accept\": \"application/json\",\n",
    "            \"Authorization\": \"Bearer {:s}\".format(token),\n",
    "        },\n",
    "        params = kwargs\n",
    "    )\n",
    "    j = x.json()\n",
    "    if 'error' in j:\n",
    "        raise Exception(\"Error {:d}, {:s}\".format(j['error']['status'], j['error']['message']))\n",
    "    return j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1704ee27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_playlist_tracks(spid, token):\n",
    "    \"\"\"\n",
    "    Obtain list of ordered tracks in a playlist,\n",
    "    discarding playlist metadata (only care about individual tracks)\n",
    "    Input: Spotify ID of playlist, OAuth token\n",
    "    Returns: list of Spotify tracks\n",
    "    \"\"\"\n",
    "    j = spquery(\"https://api.spotify.com/v1/playlists/{:s}/tracks\".format(spid), token=token)\n",
    "    return [entry['track'] for entry in j['items']]\n",
    "\n",
    "\n",
    "def get_track_audio_features(spid, token):\n",
    "    \"\"\"Returns: dict of track audio features\"\"\"\n",
    "    j = spquery(\"https://api.spotify.com/v1/audio-features/{:s}\".format(spid), token=token)\n",
    "    return j\n",
    "\n",
    "\n",
    "def get_multiple_track_audio_features(spids, token):\n",
    "    \"\"\"Returns: list of dicts with track audio features\"\"\"\n",
    "    j = spquery(\"https://api.spotify.com/v1/audio-features\", token=token, ids=\",\".join(spids))\n",
    "    return j['audio_features']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfb74f24",
   "metadata": {},
   "source": [
    "# Construct the data product we want: list of tracks, 1 dict per track holding just the most relevant/interesting audio features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3ffcd567",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset_from_playlist(spid, token):\n",
    "    \"\"\"\n",
    "    Input: spotify ID of playlist\n",
    "    Output: our custom-munged dataset\n",
    "    \"\"\"\n",
    "    tracks = get_playlist_tracks(spid, token=token)\n",
    "    ids = [track['id'] for track in tracks]\n",
    "    features = get_multiple_track_audio_features(ids, token=token)\n",
    "    \n",
    "    # attach interesting audio features\n",
    "    # https://developer.spotify.com/documentation/web-api/reference/#/operations/get-audio-features\n",
    "    wanted_features = [\n",
    "        'acousticness',\n",
    "        'danceability',\n",
    "        'energy',\n",
    "        'instrumentalness',\n",
    "        'key',\n",
    "        'liveness',\n",
    "        'loudness',\n",
    "        'mode',\n",
    "        'speechiness',\n",
    "        'tempo',\n",
    "        'time_signature',\n",
    "        'valence',\n",
    "    ]\n",
    "\n",
    "    for f, t in zip(features,tracks):\n",
    "        assert f['id'] == t['id']  # really basic sanity check\n",
    "        for k in wanted_features:\n",
    "            assert k not in t\n",
    "            t[k] = f[k]\n",
    "    \n",
    "    # pare down unnecessarily long features\n",
    "    for t in tracks:\n",
    "        \n",
    "        # keep only album name\n",
    "        album_name = t['album']['name']\n",
    "        t['album'] = album_name\n",
    "        \n",
    "        # keep only a list of artist names\n",
    "        #artists = [a['name'] for a in t['artists']]\n",
    "        #t['artists'] = artists\n",
    "        \n",
    "        # keep only the very first artist\n",
    "        artist = t['artists'][0]['name']\n",
    "        del t['artists']\n",
    "        t['artist'] = artist\n",
    "    \n",
    "    # remove unneeded features\n",
    "    # note, \"id\" is useful for unqiuely identifying song on spotify, and for further data queries,\n",
    "    # but suppress this b/c the envisioned users will not do any querying.\n",
    "    delete_keys = [\n",
    "        'available_markets',\n",
    "        'disc_number',\n",
    "        'episode',\n",
    "        'external_ids',\n",
    "        'external_urls',\n",
    "        'href',\n",
    "        'id',\n",
    "        'is_local',\n",
    "        'preview_url',\n",
    "        'track',\n",
    "        'track_number',\n",
    "        'type',\n",
    "        'uri',\n",
    "    ]\n",
    "    for t in tracks:\n",
    "        for k in delete_keys:\n",
    "            del t[k]\n",
    "    \n",
    "    return tracks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7abb6be7",
   "metadata": {},
   "source": [
    "# Save playlist data to json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "204e2e3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_50_weekly_global_id = \"37i9dQZEVXbNG2KDcFcKOF\"\n",
    "\n",
    "top_50_weekly_australia_id = \"37i9dQZEVXbK4fwx2r07XW\"\n",
    "top_50_weekly_brazil_id = \"37i9dQZEVXbKzoK95AbRy9\"\n",
    "top_50_weekly_dominican_republic_id = \"37i9dQZEVXbMPoK06pe7d6\"\n",
    "top_50_weekly_nigeria_id = \"37i9dQZEVXbLw80jjcctV1\"\n",
    "top_50_weekly_norway_id = \"37i9dQZEVXbLWYFZ5CkSvr\"\n",
    "top_50_weekly_south_korea_id = \"37i9dQZEVXbJZGli0rRP3r\"\n",
    "\n",
    "top_tracks_2021_usa = \"37i9dQZF1DXbJMiQ53rTyJ\"\n",
    "top_tracks_2020_usa = \"37i9dQZF1DXaqCgtv7ZR3L\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7f59461c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wrote viral_50_daily_usa.json\n",
      "wrote top_50_daily_usa.json\n",
      "wrote top_50_weekly_usa.json\n",
      "wrote top_hits_2021.json\n",
      "wrote top_hits_2020.json\n",
      "wrote top_hits_2019.json\n",
      "wrote top_hits_2018.json\n",
      "wrote top_hits_2017.json\n",
      "wrote top_hits_2010.json\n",
      "wrote top_hits_2000.json\n",
      "wrote top_hits_1990.json\n",
      "wrote top_hits_1980.json\n",
      "wrote top_hits_1970.json\n"
     ]
    }
   ],
   "source": [
    "for fname, spid in [\n",
    "    \n",
    "    (\"viral_50_daily_usa.json\", \"37i9dQZEVXbKuaTI1Z1Afx\"),\n",
    "    (\"top_50_daily_usa.json\", \"37i9dQZEVXbLRQDuF5jeBp\"),\n",
    "    (\"top_50_weekly_usa.json\", \"37i9dQZEVXbLp5XoPON0wI\"),\n",
    "    \n",
    "    (\"top_hits_2021.json\", \"37i9dQZF1DX18jTM2l2fJY\"),\n",
    "    (\"top_hits_2020.json\", \"2fmTTbBkXi8pewbUvG3CeZ\"),\n",
    "    (\"top_hits_2019.json\", \"37i9dQZF1DWVRSukIED0e9\"),\n",
    "    (\"top_hits_2018.json\", \"37i9dQZF1DXe2bobNYDtW8\"),\n",
    "    (\"top_hits_2017.json\", \"37i9dQZF1DWTE7dVUebpUW\"),\n",
    "    \n",
    "    # I suspect spotify curates these playlists\n",
    "    # so that immediately adjacent years probably do not have repeats,\n",
    "    # even if by chart-sampling they should...\n",
    "    #(\"top_hits_2011.json\", \"37i9dQZF1DXcagnSNtrGuJ\"),\n",
    "    \n",
    "    (\"top_hits_2010.json\", \"37i9dQZF1DXc6IFF23C9jj\"),\n",
    "    (\"top_hits_2000.json\", \"37i9dQZF1DWUZv12GM5cFk\"),\n",
    "    (\"top_hits_1990.json\", \"37i9dQZF1DX4joPVMjBCAo\"),\n",
    "    (\"top_hits_1980.json\", \"37i9dQZF1DWXbLOeOIhbc5\"),\n",
    "    (\"top_hits_1970.json\", \"37i9dQZF1DWXQyLTHGuTIz\"),\n",
    "]:\n",
    "    tracks = build_dataset_from_playlist(spid, token=token)\n",
    "    coding_club_playlist.write_dataset(fname, tracks)\n",
    "    print(\"wrote\", fname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}