Add full search and user/track/playlist cache (#900)

* Add full search and user/track/playlist cache

* Fix typo

discovery-provider/src/api/v1/api.py

@@ -5,6 +5,7 @@
 from src.api.v1.playlists import ns as playlists_ns, full_ns as full_playlists_ns
 from src.api.v1.tracks import ns as tracks_ns, full_ns as full_tracks_ns
 from src.api.v1.metrics import ns as metrics_ns
+from src.api.v1.search import full_ns as full_search_ns
 from src.api.v1.models.users import ns as models_ns
 from src.api.v1.resolve import ns as resolve_ns
 
@@ -33,3 +34,4 @@ def specs_url(self):
 api_v1_full.add_namespace(full_tracks_ns)
 api_v1_full.add_namespace(full_playlists_ns)
 api_v1_full.add_namespace(full_users_ns)
+api_v1_full.add_namespace(full_search_ns)

discovery-provider/src/api/v1/helpers.py

@@ -149,15 +149,21 @@ def parse_unix_epoch_param(time, default=0):
 def extend_track(track):
     track_id = encode_int_id(track["track_id"])
     owner_id = encode_int_id(track["owner_id"])
-    if ("user" in track):
+    if "user" in track:
         track["user"] = extend_user(track["user"])
     track["id"] = track_id
     track["user_id"] = owner_id
-    track["followee_favorites"] = list(map(extend_favorite, track["followee_saves"]))
-    track["followee_resposts"] = list(map(extend_repost, track["followee_reposts"]))
+    if "followee_saves" in track:
+        track["followee_favorites"] = list(map(extend_favorite, track["followee_saves"]))
+    if "followee_reposts" in track:
+        track["followee_reposts"] = list(map(extend_repost, track["followee_reposts"]))
+    if "remix_of" in track:
+        track["remix_of"] = extend_remix_of(track["remix_of"])
+
     track = add_track_artwork(track)
-    track["remix_of"] = extend_remix_of(track["remix_of"])
-    track["favorite_count"] = track["save_count"]
+
+    if "save_count" in track:
+        track["favorite_count"] = track["save_count"]
 
     duration = 0.
     for segment in track["track_segments"]:
@@ -189,14 +195,16 @@ def extend_playlist(playlist):
     owner_id = encode_int_id(playlist["playlist_owner_id"])
     playlist["id"] = playlist_id
     playlist["user_id"] = owner_id
-    if ("user" in playlist):
-        playlist["user"] = extend_user(playlist["user"])
     playlist = add_playlist_artwork(playlist)
+    if "user" in playlist:
+        playlist["user"] = extend_user(playlist["user"])
+    if "followee_saves" in playlist:
+        playlist["followee_favorites"] = list(map(extend_favorite, playlist["followee_saves"]))
+    if "followee_reposts" in playlist:
+        playlist["followee_reposts"] = list(map(extend_repost, playlist["followee_reposts"]))
+    if "save_count" in playlist:
+        playlist["favorite_count"] = playlist["save_count"]
 
-    playlist["followee_favorites"] = list(map(extend_favorite, playlist["followee_saves"]))
-    playlist["followee_reposts"] = list(map(extend_repost, playlist["followee_reposts"]))
-
-    playlist["favorite_count"] = playlist["save_count"]
     playlist["added_timestamps"] = add_playlist_added_timestamps(playlist)
     playlist["cover_art"] = playlist["playlist_image_multihash"]
     playlist["cover_art_sizes"] = playlist["playlist_image_sizes_multihash"]

discovery-provider/src/api/v1/models/search.py

@@ -0,0 +1,18 @@
+from flask_restx.fields import Boolean
+from src.api.v1.helpers import make_response
+from flask_restx import fields
+from .common import ns
+from .tracks import track_full
+from .users import user_model_full
+from .playlists import full_playlist_model
+
+search_model = ns.model('search_model', {
+  "users": fields.List(fields.Nested(user_model_full), required=True),
+  "followed_users": fields.List(fields.Nested(user_model_full), required=False),
+  "tracks": fields.List(fields.Nested(track_full), required=True),
+  "saved_tracks": fields.List(fields.Nested(track_full), required=False),
+  "playlists": fields.List(fields.Nested(full_playlist_model), required=True),
+  "saved_playlists": fields.List(fields.Nested(full_playlist_model), required=False),
+  "albums": fields.List(fields.Nested(full_playlist_model), required=True),
+  "saved_albums": fields.List(fields.Nested(full_playlist_model), required=False)
+})

discovery-provider/src/api/v1/search.py

@@ -0,0 +1,130 @@
+import logging  # pylint: disable=C0302
+from flask_restx import Resource, Namespace, fields, reqparse
+from src.api.v1.helpers import extend_track, make_response, get_default_max, \
+    success_response, format_offset, format_limit, decode_string_id, \
+    extend_user, extend_track, extend_playlist
+from src.queries.search_queries import SearchKind, search
+from src.utils.redis_cache import cache, extract_key, use_redis_cache
+from src.utils.redis_metrics import record_metrics
+from src.api.v1.models.search import search_model
+
+logger = logging.getLogger(__name__)
+
+# Models & namespaces
+
+full_ns = Namespace('search', description='Full search operations')
+
+# Helpers
+def extend_search(resp):
+    if 'users' in resp:
+        resp['users'] = list(map(extend_user, resp['users']))
+    if 'followed_users' in resp:
+        resp['followed_users'] = list(map(extend_user, resp['followed_users']))
+    if 'tracks' in resp:
+        resp['tracks'] = list(map(extend_track, resp['tracks']))
+    if 'saved_tracks' in resp:
+        resp['saved_tracks'] = list(map(extend_track, resp['saved_tracks']))
+    if 'playlists' in resp:
+        resp['playlists'] = list(map(extend_playlist, resp['playlists']))
+    if 'saved_playlists' in resp:
+        resp['saved_playlists'] = list(map(extend_playlist, resp['saved_playlists']))
+    if 'albums' in resp:
+        resp['albums'] = list(map(extend_playlist, resp['albums']))
+    if 'saved_albums' in resp:
+        resp['saved_albums'] = list(map(extend_playlist, resp['saved_albums']))
+    return resp
+
+search_route_parser = reqparse.RequestParser()
+search_route_parser.add_argument('user_id', required=False)
+search_route_parser.add_argument(
+    'kind', required=False, type=str, default='all', choices=('all', 'users', 'tracks', 'playlists', 'albums'))
+search_route_parser.add_argument('query', required=True, type=str)
+search_route_parser.add_argument('limit', required=False, type=int)
+search_route_parser.add_argument('offset', required=False, type=int)
+search_full_response = make_response("search_full_response", full_ns, fields.Nested(search_model))
+@full_ns.route("/full")
+class FullSearch(Resource):
+    @full_ns.expect(search_route_parser)
+    @full_ns.doc(
+        id="""Get Users/Tracks/Playlists/Albums that best match the search query""",
+        params={
+            'user_id': 'A User ID of the requesting user to personalize the response',
+            'query': 'Search query text',
+            'kind': 'The type of response, one of: all, users, tracks, playlists, or albums',
+            'limit': 'Limit',
+            'offset': 'Offset'
+        },
+        responses={
+            200: 'Success',
+            400: 'Bad request',
+            500: 'Server error'
+        }
+    )
+    @full_ns.marshal_with(search_full_response)
+    @cache(ttl_sec=5)
+    def get(self):
+        args = search_route_parser.parse_args()
+        limit = get_default_max(args.get('limit'), 10, 100)
+        offset = get_default_max(args.get('offset'), 0)
+
+        current_user_id = None
+        if args.get("user_id"):
+            current_user_id = decode_string_id(args["user_id"])
+        search_args = {
+          "is_auto_complete": False,
+          "kind": args.get("kind", "all"),
+          "query": args.get("query"),
+          "current_user_id": current_user_id,
+          "with_users": True,
+          "limit": limit,
+          "offset": offset,
+          "only_downloadable": False
+        }
+        resp = search(search_args)
+        resp = extend_search(resp)
+
+        return success_response(resp)
+
+search_autocomplete_response = make_response("search_autocomplete_response", full_ns, fields.Nested(search_model))
+@full_ns.route("/autocomplete")
+class FullSearch(Resource):
+    @full_ns.expect(search_route_parser)
+    @full_ns.doc(
+        id="""Get Users/Tracks/Playlists/albums that best match the search query""",
+        params={
+            'user_id': 'A User ID of the requesting user to personalize the response',
+            'query': 'Search query text',
+            'kind': 'The type of response, one of: all, users, tracks, playlists, or albums',
+            'limit': 'Limit',
+            'offset': 'Offset'
+        },
+        responses={
+            200: 'Success',
+            400: 'Bad request',
+            500: 'Server error'
+        }
+    )
+    @full_ns.marshal_with(search_autocomplete_response)
+    @cache(ttl_sec=5)
+    def get(self):
+        args = search_route_parser.parse_args()
+        limit = get_default_max(args.get('limit'), 10, 100)
+        offset = get_default_max(args.get('offset'), 0)
+
+        current_user_id = None
+        if args.get("user_id"):
+            current_user_id = decode_string_id(args["user_id"])
+        search_args = {
+          "is_auto_complete": True,
+          "kind": args.get("kind", "all"),
+          "query": args.get("query"),
+          "current_user_id": current_user_id,
+          "with_users": False,
+          "limit": limit,
+          "offset": offset,
+          "only_downloadable": False
+        }
+        resp = search(search_args)
+        resp = extend_search(resp)
+
+        return success_response(resp)

discovery-provider/src/queries/get_feed.py

@@ -6,6 +6,7 @@
 from src.utils import helpers
 from src.utils.db_session import get_db_read_replica
 from src.queries import response_name_constants
+from src.queries.get_unpopulated_tracks import get_unpopulated_tracks
 from src.queries.query_helpers import get_current_user_id, populate_track_metadata, \
     populate_playlist_metadata, get_pagination_vars, paginate_query, get_users_by_id, get_users_ids
 
@@ -60,14 +61,7 @@ def get_feed(args):
                         playlist_track_ids.add(track["track"])
 
                 # get all track objects for track ids
-                playlist_tracks = (
-                    session.query(Track)
-                    .filter(
-                        Track.is_current == True,
-                        Track.track_id.in_(playlist_track_ids)
-                    )
-                    .all()
-                )
+                playlist_tracks = get_unpopulated_tracks(session, playlist_track_ids)
                 playlist_tracks_dict = {
                     track.track_id: track for track in playlist_tracks}
 

discovery-provider/src/queries/get_followees_for_user.py

@@ -1,11 +1,11 @@
 from sqlalchemy import func, desc
 from sqlalchemy.orm import aliased
 
-from src.models import User, Follow
-from src.utils import helpers
+from src.models import Follow
 from src.utils.db_session import get_db_read_replica
 from src.queries import response_name_constants
 from src.queries.query_helpers import populate_user_metadata, add_query_pagination
+from src.queries.get_unpopulated_users import get_unpopulated_users
 
 def get_followees_for_user(args):
     users = []
@@ -55,15 +55,7 @@ def get_followees_for_user(args):
                     in followee_user_ids_by_follower_count]
 
         # get all users for above user_ids
-        users = (
-            session.query(User)
-            .filter(
-                User.is_current == True,
-                User.user_id.in_(user_ids)
-            )
-            .all()
-        )
-        users = helpers.query_result_to_list(users)
+        users = get_unpopulated_users(session, user_ids)
 
         # bundle peripheral info into user results
         users = populate_user_metadata(

discovery-provider/src/queries/get_followers_for_user.py

@@ -1,11 +1,11 @@
 from sqlalchemy import func, asc, desc
 from sqlalchemy.orm import aliased
 
-from src.models import User, Follow
-from src.utils import helpers
+from src.models import Follow
 from src.utils.db_session import get_db_read_replica
 from src.queries import response_name_constants
 from src.queries.query_helpers import populate_user_metadata, add_query_pagination
+from src.queries.get_unpopulated_users import get_unpopulated_users
 
 def get_followers_for_user(args):
     users = []
@@ -60,15 +60,7 @@ def get_followers_for_user(args):
                     in follower_user_ids_by_follower_count]
 
         # get all users for above user_ids
-        users = (
-            session.query(User)
-            .filter(
-                User.is_current == True,
-                User.user_id.in_(user_ids)
-            )
-            .all()
-        )
-        users = helpers.query_result_to_list(users)
+        users = get_unpopulated_users(session, user_ids)
 
         # bundle peripheral info into user results
         users = populate_user_metadata(

discovery-provider/src/queries/get_remixes_of.py

@@ -5,6 +5,7 @@
 from src.models import Track, Repost, RepostType, Save, SaveType, Remix
 from src.utils import helpers
 from src.utils.db_session import get_db_read_replica
+from src.queries.get_unpopulated_tracks import get_unpopulated_tracks
 from src.queries.query_helpers import populate_track_metadata, \
     add_query_pagination, add_users_to_tracks, create_save_count_subquery, \
     create_repost_count_subquery
@@ -28,16 +29,15 @@ def get_remixes_of(args):
 
     with db.scoped_session() as session:
         def get_unpopulated_remixes():
+
             # Fetch the parent track to get the track's owner id
-            parent_track = session.query(Track).filter(
-                Track.is_current == True,
-                Track.track_id == track_id
-            ).first()
+            parent_track_res = get_unpopulated_tracks(session, [track_id])
 
-            if parent_track == None:
+            if not parent_track_res or parent_track_res[0] is None:
                 raise exceptions.ArgumentError("Invalid track_id provided")
 
-            track_owner_id = parent_track.owner_id
+            parent_track = parent_track_res[0]
+            track_owner_id = parent_track['owner_id']
 
             # Create subquery for save counts for sorting
             save_count_subquery = create_save_count_subquery(

discovery-provider/src/queries/get_top_genre_users.py

@@ -2,9 +2,9 @@
 from sqlalchemy import func, asc, desc
 
 from src.models import User, Track, Follow
-from src.utils import helpers
 from src.utils.db_session import get_db_read_replica
 from src.queries.query_helpers import populate_user_metadata, paginate_query
+from src.queries.get_unpopulated_users import get_unpopulated_users
 
 logger = logging.getLogger(__name__)
 
@@ -92,12 +92,7 @@ def get_top_genre_users(args):
 
         # If the with_users flag is used, retrieve the user metadata
         if with_users:
-            user_query = session.query(User).filter(
-                User.user_id.in_(user_ids),
-                User.is_current == True
-            )
-            users = user_query.all()
-            users = helpers.query_result_to_list(users)
+            users = get_unpopulated_users(session, user_ids)
             queried_user_ids = list(map(lambda user: user["user_id"], users))
             users = populate_user_metadata(
                 session, queried_user_ids, users, None)

discovery-provider/src/queries/get_tracks_including_unlisted.py

@@ -92,7 +92,8 @@ def filter_fn(track):
             return (tracks, track_ids)
 
         key = make_cache_key(args)
-        (tracks, track_ids) = use_redis_cache(key, UNPOPULATED_TRACK_CACHE_DURATION_SEC, get_unpopulated_track)
+        (tracks, track_ids) = use_redis_cache(
+            key, UNPOPULATED_TRACK_CACHE_DURATION_SEC, get_unpopulated_track)
 
         # Add users
         if args.get("with_users", False):

discovery-provider/src/queries/get_trending_tracks.py

@@ -3,9 +3,8 @@
 from dateutil.parser import parse
 from flask.globals import request
 
-from src.models import Track
-from src.utils import helpers
 from src.utils.db_session import get_db_read_replica
+from src.queries.get_unpopulated_tracks import get_unpopulated_tracks
 from src.queries.query_helpers import populate_track_metadata, \
     get_users_ids, get_users_by_id
 from src.tasks.generate_trending import generate_trending
@@ -65,13 +64,7 @@ def get_unpopulated_trending():
 
             track_ids = [track['track_id'] for track in sorted_track_scores]
 
-            tracks = session.query(Track).filter(
-                Track.is_current == True,
-                Track.is_unlisted == False,
-                Track.stem_of == None,
-                Track.track_id.in_(track_ids)
-            ).all()
-            tracks = helpers.query_result_to_list(tracks)
+            tracks = get_unpopulated_tracks(session, track_ids)
             return (tracks, track_ids)
 
         # get scored trending tracks, either