Skip to content

Aggregator and stats update #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TheTedLab merged 3 commits into from
Dec 10, 2023
Merged

Aggregator and stats update #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
366b65d
Update aggregator
TheTedLab Nov 27, 2023
4d40c2d
Update artist saving and docker
TheTedLab Dec 7, 2023
4f59e91
Add updating for playcount and stats
TheTedLab Dec 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/aggregator/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ RUN pip install -r /app/requirements.txt
# Copy files to the container
COPY save_artist.py .
COPY server_vars.py .
COPY resources/artists/artist-1.json ./resources/artists/artist-1.json
COPY resources/artists/ ./resources/artists/

# Command to run application
CMD ["python", "save_artist.py"]
182 changes: 139 additions & 43 deletions src/aggregator/aggregator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import time
import requests
import datetime
from auth_credentials import client_id, client_secret, client_headers, client_extensions, token_type, access_token
from auth_credentials import client_id, client_secret, token_type, access_token
from classes.artist import Artist
from classes.album import Album
from classes.track import Track
Expand All @@ -15,12 +15,9 @@


def main():

print(f' [T] Time: {datetime.datetime.now()}')

timeout = 0.5

request_count = 0
timeout, request_count = 2, 0

headers = {
'Authorization': f'{token_type} {access_token}',
Expand All @@ -29,17 +26,16 @@ def main():
with open(f'resources/artists-ids-list.json', 'r', encoding='utf-8') as file:
artists_ids = json.load(file)

for artist_id in artists_ids[10392:10393]:
for artist_id in artists_ids[773:1001]:
# 1 request
response = requests.get(
f'https://api.spotify.com/v1/artists/{artist_id}',
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

print(response.status_code)
if response.status_code != 200:
print('Token expired!')
# 2 request (potential)
Expand All @@ -49,7 +45,7 @@ def main():
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

headers['Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

Expand All @@ -60,9 +56,10 @@ def main():
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

artist_info_data = response.json()
print(f"Artist ID: {artist_info_data.get('id')}")
print(f"Artist Name: {artist_info_data.get('name')}")
print(f"Artist Popularity: {artist_info_data.get('popularity')}")
print(f"Artist Genres: {artist_info_data.get('genres')}")
Expand All @@ -76,44 +73,44 @@ def main():
artist_info_data.get('genres'),
)

# client_params = {
# 'operationName': 'queryArtistOverview',
# 'variables': '{"uri":"spotify:artist:' + artist_id + '","locale":"","includePrerelease":true}',
# 'extensions': client_extensions
# }
#
# 4 request
# response = requests.get(
# 'https://api-partner.spotify.com/pathfinder/v1/query',
# params=client_params,
# headers=client_headers
# )
# time.sleep(timeout)
# request_count += 1
# print(f" [*] Request {request_count} - {response.status_code}]")
#
# artist_stats_data = response.json().get('data').get('artistUnion').get('stats')
# print(f"Monthly Listeners: {artist_stats_data.get('monthlyListeners')}")
# print(f"World Rank: {artist_stats_data.get('worldRank')}")
# print(f"Cities: {artist_stats_data.get('topCities').get('items')}")
# print()

album_params = {
'include_groups': 'album',
'limit': 50,
'offset': 0,
'market': 'ES'
}

# 5 request
# 4 request
response = requests.get(
f'https://api.spotify.com/v1/artists/{artist_id}/albums',
params=album_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

if response.status_code != 200:
print('Token expired!')

response = requests.post(
'https://accounts.spotify.com/api/token',
data=token_data
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

headers['Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

response = requests.get(
f'https://api.spotify.com/v1/artists/{artist_id}/albums',
params=album_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

artist_albums_ids = list()
artist_albums = response.json().get('items')
Expand All @@ -122,15 +119,37 @@ def main():

album_params['include_groups'] = 'single'

# 6 request
# 5 request
response = requests.get(
f'https://api.spotify.com/v1/artists/{artist_id}/albums',
params=album_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

if response.status_code != 200:
print('Token expired!')

response = requests.post(
'https://accounts.spotify.com/api/token',
data=token_data
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

headers['Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

response = requests.get(
f'https://api.spotify.com/v1/artists/{artist_id}/albums',
params=album_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

artist_albums = response.json().get('items')
for album in artist_albums:
Expand All @@ -143,7 +162,7 @@ def main():
'ids': ''
}

# 6 requests above, 54 requests below left
# 5 requests above, 55 requests below left
albums_json = list()
tracks_json = list()
id_offset = 20
Expand All @@ -157,7 +176,29 @@ def main():
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(f" [*] [Request {request_count} - {response.status_code}]")

if response.status_code != 200:
print('Token expired!')

response = requests.post(
'https://accounts.spotify.com/api/token',
data=token_data
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

headers['Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

response = requests.get(
'https://api.spotify.com/v1/albums',
params=several_albums_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(f" [*] [Request {request_count} - {response.status_code}]")

album_data = response.json().get('albums')
for album in album_data:
Expand Down Expand Up @@ -228,7 +269,33 @@ def main():
)
time.sleep(timeout)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(
f" [*] Request {request_count} - {response.status_code}]")

if response.status_code != 200:
print('Token expired!')

response = requests.post(
'https://accounts.spotify.com/api/token',
data=token_data
)
time.sleep(timeout)
request_count += 1
print(
f" [*] Request {request_count} - {response.status_code}]")

headers[
'Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

response = requests.get(
'https://api.spotify.com/v1/tracks',
params=several_tracks_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(
f" [*] Request {request_count} - {response.status_code}]")

tracks_data = response.json().get('tracks')
for track_obj in tracks_data:
Expand All @@ -252,12 +319,43 @@ def main():
params=several_features_params,
headers=headers
)
time.sleep(timeout)
time.sleep(timeout + 1)
request_count += 1
print(f" [*] Request {request_count} - {response.status_code}]")
print(
f" [*] Request {request_count} - {response.status_code}]")

if response.status_code != 200:
print('Token expired!')

response = requests.post(
'https://accounts.spotify.com/api/token',
data=token_data
)
time.sleep(timeout)
request_count += 1
print(
f" [*] Request {request_count} - {response.status_code}]")

headers[
'Authorization'] = f"{response.json().get('token_type')} {response.json().get('access_token')}"

response = requests.get(
'https://api.spotify.com/v1/audio-features',
params=several_features_params,
headers=headers
)
time.sleep(timeout)
request_count += 1
print(
f" [*] Request {request_count} - {response.status_code}]")

features_data = response.json().get('audio_features')
for features_track in features_data:
try:
features_track.get('id')
except AttributeError:
continue

print(f"Track ID: {features_track.get('id')}")
print(f"Track Acousticness: {features_track.get('acousticness')}")
print(f"Track Danceability: {features_track.get('danceability')}")
Expand Down Expand Up @@ -291,9 +389,7 @@ def main():
artist_for_json.albums = albums_json
artist_for_json.tracks = tracks_json

# print(artist_for_json)
json_string = json.dumps(artist_for_json, indent=4, ensure_ascii=False, default=lambda x: x.__dict__)
# print(json_string)
with open(f'resources/artists/artist-{artist_id}.json', 'w', encoding='utf-8') as file:
file.write(json_string)

Expand Down
3 changes: 2 additions & 1 deletion src/aggregator/auth_credentials.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ client_headers = {
'your': 'client_headers'
}

client_extensions = '{"your_client_extensions"}'
get_artist_stats_extensions = '{"your_client_extensions_for_artist_stats_request"}'
get_tracks_extensions = '{"your_client_extensions_for_get_album_request"}'
Loading