Fix code inspection errors

src/aggregator/save_artist.py

@@ -27,3 +27,15 @@
         )
         print(f"{response.text} - Status Code: {response.status_code}")
         time.sleep(1)
+
+        if response.status_code != 200:
+            print(f"Status Code: {response.status_code}, Retrying...")
+            time.sleep(5)
+            response = requests.post(
+                f'http://{SERVER_ADDRESS}:{SERVER_PORT}/saveArtist',
+                headers=header,
+                data=json_string,
+                timeout=10
+            )
+            print(f"{response.text} - Status Code: {response.status_code}")
+            time.sleep(1)

src/aggregator/stats_from_files.py

@@ -1,34 +1,25 @@
 """Module for adding artists stats."""
 import datetime
 import json
-from os import listdir
-from os.path import isfile, join
 
 from src.aggregator.stats_utils import (
     change_artist_data,
     change_track_data,
-    get_artist_response_template
+    get_artist_response_template,
+    get_artist_albums_ids,
+    get_ids_from_file_names
 )
 
 
 def stats_from_files_main(
-        artists_path, timeout, request_count,
+        timeout, request_count,
         file_path, artist_count, albums_path,
         headers, extensions
 ):
     """Main function for stats data aggregation."""
     print(f'[T] Time: {datetime.datetime.now()}')
 
-    artists_files_list = listdir(artists_path)
-    # artists_files_list.remove('.DS_Store')
-    artist_ids = [
-        file.split('.')[0].split('-')[1] for file in artists_files_list if isfile(
-            join(
-                artists_path,
-                file
-            )
-        )
-    ]
+    artist_ids = get_ids_from_file_names(file_path)
 
     for artist_id in artist_ids[artist_count:]:
         response, request_count = get_artist_response_template(
@@ -44,21 +35,13 @@ def stats_from_files_main(
             f" - {response.status_code}] - Number {artist_count}"
         )
 
-        albums_ids = change_artist_data(
-            response, artist_id, file_path
-        )
+        change_artist_data(response, artist_id, file_path)
+
+        albums_ids = get_artist_albums_ids(artist_id, file_path)
+
+        artist_albums = get_ids_from_file_names(albums_path)
 
         for album_id in albums_ids:
-            artist_albums = [
-                file.split('.')[0].split('-')[1] for file in listdir(
-                    albums_path
-                ) if isfile(
-                    join(
-                        albums_path,
-                        file
-                    )
-                )
-            ]
             if album_id in artist_albums:
                 with open(
                         f'{albums_path}/album-{album_id}.json',
@@ -67,17 +50,12 @@ def stats_from_files_main(
                 ) as album_file:
                     album_data = json.load(album_file)
 
-                    tracks_data = album_data.get(
-                        'data'
-                    ).get('albumUnion').get('tracks').get('items')
-
-                    change_track_data(tracks_data, artist_id, file_path)
+                    change_track_data(album_data, artist_id, file_path)
         artist_count += 1
 
 
 # if __name__ == '__main__':
 #     stats_from_files_main(
-#         artists_path="src/aggregator/resources/artists",
 #         timeout=1,
 #         request_count=0,
 #         file_path="src/aggregator/resources/artists",

src/aggregator/stats_update.py

@@ -7,7 +7,8 @@
 from src.aggregator.stats_utils import (
     change_artist_data,
     change_track_data,
-    get_artist_response_template
+    get_artist_response_template,
+    get_artist_albums_ids
 )
 
 
@@ -27,9 +28,9 @@ def stats_update_main(
         )
         print(f"Artist Stats {artist_id}  - [*] [Request {request_count} - {response.status_code}]")
 
-        albums_ids = change_artist_data(
-            response, artist_id, file_path
-        )
+        change_artist_data(response, artist_id, file_path)
+
+        albums_ids = get_artist_albums_ids(artist_id, file_path)
 
         for album_id in albums_ids:
             get_album_params = {
@@ -51,9 +52,7 @@ def stats_update_main(
             print(f"Get Album {album_id}"
                   f"     - [*] [Request {request_count} - {response.status_code}]")
 
-            tracks_data = response.json().get('data').get('albumUnion').get('tracks').get('items')
-
-            change_track_data(tracks_data, artist_id, file_path)
+            change_track_data(response.json(), artist_id, file_path)
 
 
 # if __name__ == '__main__':

src/aggregator/stats_utils.py

@@ -2,11 +2,13 @@
 import json
 import time
 from collections import OrderedDict
+from os import listdir
+from os.path import isfile, join
 
 import requests
 
 
-def change_artist_data(response: requests.Response, artist_id, file_path):
+def change_artist_data(response, artist_id, file_path):
     """Util function for artists stats data aggregation or updating."""
     artist_stats = response.json().get('data').get('artistUnion').get('stats')
 
@@ -30,23 +32,11 @@ def change_artist_data(response: requests.Response, artist_id, file_path):
         )
         artist_file.truncate()
 
-    with open(
-            f'{file_path}/artist-{artist_id}.json',
-            mode='r',
-            encoding='utf-8'
-    ) as artist_file:
-        data = json.load(artist_file)
-
-    artist_albums = data.get('albums')
-    albums_ids = []
-    for album in artist_albums:
-        albums_ids.append(album.get('album_id'))
-
-    return albums_ids
-
 
-def change_track_data(tracks_data, artist_id, file_path):
+def change_track_data(response, artist_id, file_path):
     """Util function for track stats data aggregation or updating."""
+    tracks_data = response.get('data').get('albumUnion').get('tracks').get('items')
+
     for track in tracks_data:
         track_data = track.get('track')
         track_id = track_data.get('uri').split(':')[2]
@@ -97,6 +87,40 @@ def get_artist_response_template(artist_id, timeout, request_count, headers, ext
     return response, request_count
 
 
+def get_artist_albums_ids(artist_id, file_path):
+    """Util function for taking artist albums ids."""
+    with open(
+            f'{file_path}/artist-{artist_id}.json',
+            mode='r',
+            encoding='utf-8'
+    ) as artist_file:
+        data = json.load(artist_file)
+
+    artist_albums = data.get('albums')
+    albums_ids = []
+    for album in artist_albums:
+        albums_ids.append(album.get('album_id'))
+
+    return albums_ids
+
+
+def get_ids_from_file_names(files_path):
+    """Util function for taking ids from files names by files path."""
+    files_list = listdir(files_path)
+    if '.DS_Store' in files_list:
+        files_list.remove('.DS_Store')
+    files_ids = [
+        file.split('.')[0].split('-')[1] for file in files_list if isfile(
+            join(
+                files_path,
+                file
+            )
+        )
+    ]
+
+    return files_ids
+
+
 # if __name__ == '__main__':
 #     response, request_count = get_artist_response_template(
 #         artist_id='0M2HHtY3OOQzIZxrHkbJLT',

tests/test_unit.py

@@ -487,7 +487,7 @@ def test_change_artist_data(requests_mock, get_artist_json_data, artist_id):
     """Test change_artist_data function with artist_id parameter of two positive ids."""
     requests_mock.get(url='https://localhost:8080', json=get_artist_json_data)
     response = requests.get(url='https://localhost:8080')
-    actual_ids = change_artist_data(
+    change_artist_data(
         response=response,
         artist_id=artist_id,
         file_path="tests/resources/artists"
@@ -507,12 +507,6 @@ def test_change_artist_data(requests_mock, get_artist_json_data, artist_id):
     assert actual_data.get('world_rank') == expected_data.get('worldRank')
     assert actual_data.get('top_cities') == expected_data.get('topCities').get('items')
 
-    expected_ids = []
-    for album in actual_data.get('albums'):
-        expected_ids.append(album.get('album_id'))
-
-    assert actual_ids == expected_ids
-
 
 @pytest.mark.unit
 def test_change_artist_data_invalid_exception(requests_mock, get_artist_json_data):
@@ -548,9 +542,8 @@ def test_change_artist_data_empty_exception(requests_mock, get_artist_json_data)
 @pytest.mark.parametrize('artist_id', ['0M2HHtY3OOQzIZxrHkbJLT'])
 def test_change_track_data(get_tracks_json_data, artist_id):
     """Test change_track_data function with artist_id parameter of one positive id."""
-    tracks_data = get_tracks_json_data.get('data').get('albumUnion').get('tracks').get('items')
     change_track_data(
-        tracks_data=tracks_data,
+        response=get_tracks_json_data,
         artist_id=artist_id,
         file_path='tests/resources/artists'
     )
@@ -573,10 +566,9 @@ def test_change_track_data(get_tracks_json_data, artist_id):
 @pytest.mark.unit
 def test_change_track_data_invalid_exception(get_tracks_json_data):
     """Test change_track_data function with artist_id parameter of invalid id."""
-    tracks_data = get_tracks_json_data.get('data').get('albumUnion').get('tracks').get('items')
     with pytest.raises(json.JSONDecodeError) as excinfo:
         change_track_data(
-            tracks_data=tracks_data,
+            response=get_tracks_json_data,
             artist_id="2kK21234",
             file_path="tests/resources"
         )
@@ -587,10 +579,9 @@ def test_change_track_data_invalid_exception(get_tracks_json_data):
 @pytest.mark.unit
 def test_change_track_data_empty_exception(get_tracks_json_data):
     """Test change_track_data function with artist_id parameter of empty id."""
-    tracks_data = get_tracks_json_data.get('data').get('albumUnion').get('tracks').get('items')
     with pytest.raises(FileNotFoundError) as excinfo:
         change_track_data(
-            tracks_data=tracks_data,
+            response=get_tracks_json_data,
             artist_id="",
             file_path="tests/resources"
         )
@@ -740,7 +731,6 @@ def test_stats_from_files_main(
     )
 
     stats_from_files_main(
-        artists_path="tests/resources/artists",
         timeout=1,
         request_count=0,
         file_path="tests/resources/artists",
@@ -794,7 +784,6 @@ def test_stats_from_files_main_invalid_exception(
 
     with pytest.raises(json.JSONDecodeError) as excinfo:
         stats_from_files_main(
-            artists_path="tests/resources/artist-test",
             timeout=1,
             request_count=0,
             file_path="tests/resources/artist-test",
@@ -820,7 +809,6 @@ def test_stats_from_files_main_empty_exception(
 
     with pytest.raises(FileNotFoundError) as excinfo:
         stats_from_files_main(
-            artists_path="tests/resources/artist",
             timeout=1,
             request_count=0,
             file_path="tests/resources/artist",
@@ -830,4 +818,5 @@ def test_stats_from_files_main_empty_exception(
             extensions=''
         )
     exception_msg = excinfo.value.args[1]
+    # assert exception_msg == 'Системе не удается найти указанный путь'
     assert exception_msg == 'No such file or directory'