Add new trending strategy for all time scores (#2362)

* Add new trending strategy for all time scores

* drop extraneous check

discovery-provider/integration_tests/tasks/test_update_trending.py

@@ -4,8 +4,8 @@
 from integration_tests.utils import populate_mock_db
 from src.models import AggregateIntervalPlay, TrackTrendingScore, TrendingParam
 from src.tasks.index_aggregate_plays import _update_aggregate_plays
-from src.trending_strategies.ML51L_trending_tracks_strategy import (
-    TrendingTracksStrategyML51L,
+from src.trending_strategies.EJ57D_trending_tracks_strategy import (
+    TrendingTracksStrategyEJ57D,
 )
 from src.utils.db_session import get_db
 
@@ -343,7 +343,7 @@ def test_update_track_score_query(app):
 
     # setup
     setup_trending(db)
-    udpated_strategy = TrendingTracksStrategyML51L()
+    udpated_strategy = TrendingTracksStrategyEJ57D()
 
     with db.scoped_session() as session:
         session.execute("REFRESH MATERIALIZED VIEW aggregate_track")
@@ -365,11 +365,11 @@ def get_time_sorted(time_range):
 
         week_scores = get_time_sorted("week")
         month_scores = get_time_sorted("month")
-        year_scores = get_time_sorted("year")
+        all_time_scores = get_time_sorted("allTime")
 
         assert len(week_scores) == 7
         assert len(month_scores) == 7
-        assert len(year_scores) == 7
+        assert len(all_time_scores) == 7
 
         # Check that the type and version fields are correct
         for score in scores:

discovery-provider/src/queries/get_trending_tracks.py

@@ -12,7 +12,10 @@
 from src.tasks.generate_trending import generate_trending
 from src.trending_strategies.base_trending_strategy import BaseTrendingStrategy
 from src.trending_strategies.trending_strategy_factory import DEFAULT_TRENDING_VERSIONS
-from src.trending_strategies.trending_type_and_version import TrendingType
+from src.trending_strategies.trending_type_and_version import (
+    TrendingType,
+    TrendingVersion,
+)
 from src.utils.db_session import get_db_read_replica
 from src.utils.redis_cache import use_redis_cache
 
@@ -55,6 +58,12 @@ def generate_unpopulated_trending_from_mat_views(
     session, genre, time_range, strategy, limit=TRENDING_LIMIT
 ):
 
+    # use all time instead of year for version EJ57D
+    if strategy.version == TrendingVersion.EJ57D and time_range == "year":
+        time_range = "allTime"
+    elif strategy.version != TrendingVersion.EJ57D and time_range == "allTime":
+        time_range = "year"
+
     trending_track_ids_query = session.query(
         TrackTrendingScore.track_id, TrackTrendingScore.score
     ).filter(
@@ -116,7 +125,7 @@ def _get_trending_tracks_with_session(
         args.get("genre"),
         args.get("time", "week"),
     )
-    time_range = "week" if time not in ["week", "month", "year"] else time
+    time_range = "week" if time not in ["week", "month", "year", "allTime"] else time
     key = make_trending_cache_key(time_range, genre, strategy.version)
 
     # Will try to hit cached trending from task, falling back

discovery-provider/src/queries/query_helpers.py

@@ -835,18 +835,11 @@ def get_karma(
             .select_from(saves_and_reposts)
             .join(User, saves_and_reposts.c.user_id == User.user_id)
         )
-        if strategy == TrendingVersion.ML51L:
-            saves_and_reposts = saves_and_reposts.filter(
-                or_(User.cover_photo != None, User.cover_photo_sizes != None),
-                or_(User.profile_picture != None, User.profile_picture_sizes != None),
-                User.bio != None,
-            )
-        else:
-            saves_and_reposts = saves_and_reposts.filter(
-                User.cover_photo != None,
-                User.profile_picture != None,
-                User.bio != None,
-            )
+        saves_and_reposts = saves_and_reposts.filter(
+            or_(User.cover_photo != None, User.cover_photo_sizes != None),
+            or_(User.profile_picture != None, User.profile_picture_sizes != None),
+            User.bio != None,
+        )
         saves_and_reposts = saves_and_reposts.subquery()
 
     query = (

discovery-provider/src/trending_strategies/EJ57D_trending_playlists_strategy.py

@@ -0,0 +1,17 @@
+from src.trending_strategies.base_trending_strategy import BaseTrendingStrategy
+from src.trending_strategies.EJ57D_trending_tracks_strategy import z
+from src.trending_strategies.trending_type_and_version import (
+    TrendingType,
+    TrendingVersion,
+)
+
+
+class TrendingPlaylistsStrategyEJ57D(BaseTrendingStrategy):
+    def __init__(self):
+        super().__init__(TrendingType.PLAYLISTS, TrendingVersion.EJ57D)
+
+    def get_track_score(self, time_range, track):
+        return z(time_range, track)
+
+    def get_score_params(self):
+        return {"zq": 1000, "xf": True, "pt": 0, "mt": 3}

discovery-provider/src/trending_strategies/EJ57D_trending_tracks_strategy.py

@@ -0,0 +1,163 @@
+import logging
+import time
+from datetime import datetime
+
+from dateutil.parser import parse
+from sqlalchemy.sql import text
+from src.trending_strategies.base_trending_strategy import BaseTrendingStrategy
+from src.trending_strategies.trending_type_and_version import (
+    TrendingType,
+    TrendingVersion,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Trending Parameters
+N = 1
+a = max
+M = pow
+F = 50
+O = 1
+R = 0.25
+i = 0.01
+q = 100000.0
+T = {"day": 1, "week": 7, "month": 30, "year": 365, "allTime": 100000}
+y = 3
+
+
+def z(time, track):
+    # pylint: disable=W,C,R
+    E = track["listens"]
+    e = track["windowed_repost_count"]
+    t = track["repost_count"]
+    x = track["windowed_save_count"]
+    A = track["save_count"]
+    o = track["created_at"]
+    l = track["owner_follower_count"]
+    j = track["karma"]
+    if l < y:
+        return {"score": 0, **track}
+    H = (N * E + F * e + O * x + R * t + i * A) * j
+    L = T[time]
+    K = datetime.now()
+    w = parse(o)
+    k = (K - w).days
+    Q = 1
+    if k > L:
+        Q = a((1.0 / q), (M(q, (1 - k / L))))
+    return {"score": H * Q, **track}
+
+
+class TrendingTracksStrategyEJ57D(BaseTrendingStrategy):
+    def __init__(self):
+        super().__init__(TrendingType.TRACKS, TrendingVersion.EJ57D, True)
+
+    def get_track_score(self, time_range, track):
+        logger.error(
+            f"get_track_score not implemented for Trending Tracks Strategy with version {TrendingVersion.EJ57D}"
+        )
+
+    def update_track_score_query(self, session):
+        start_time = time.time()
+        trending_track_query = text(
+            """
+            begin;
+                DELETE FROM track_trending_scores WHERE type=:type AND version=:version;
+                INSERT INTO track_trending_scores
+                    (track_id, genre, type, version, time_range, score, created_at)
+                    select
+                        tp.track_id,
+                        tp.genre,
+                        :type,
+                        :version,
+                        :week_time_range,
+                        CASE
+                        WHEN tp.owner_follower_count < :y
+                            THEN 0
+                        WHEN EXTRACT(DAYS from now() - aip.created_at) > :week
+                            THEN greatest(1.0/:q, pow(:q, greatest(-10, 1.0 - 1.0*EXTRACT(DAYS from now() - aip.created_at)/:week))) * (:N * aip.week_listen_counts + :F * tp.repost_week_count + :O * tp.save_week_count + :R * tp.repost_count + :i * tp.save_count) * tp.karma
+                        ELSE (:N * aip.week_listen_counts + :F * tp.repost_week_count + :O * tp.save_week_count + :R * tp.repost_count + :i * tp.save_count) * tp.karma
+                        END as week_score,
+                        now()
+                    from trending_params tp
+                    inner join aggregate_interval_plays aip
+                        on tp.track_id = aip.track_id;
+                INSERT INTO track_trending_scores
+                    (track_id, genre, type, version, time_range, score, created_at)
+                    select
+                        tp.track_id,
+                        tp.genre,
+                        :type,
+                        :version,
+                        :month_time_range,
+                        CASE
+                        WHEN tp.owner_follower_count < :y
+                            THEN 0
+                        WHEN EXTRACT(DAYS from now() - aip.created_at) > :month
+                            THEN greatest(1.0/:q, pow(:q, greatest(-10, 1.0 - 1.0*EXTRACT(DAYS from now() - aip.created_at)/:month))) * (:N * aip.month_listen_counts + :F * tp.repost_month_count + :O * tp.save_month_count + :R * tp.repost_count + :i * tp.save_count) * tp.karma
+                        ELSE (:N * aip.month_listen_counts + :F * tp.repost_month_count + :O * tp.save_month_count + :R * tp.repost_count + :i * tp.save_count) * tp.karma
+                        END as month_score,
+                        now()
+                    from trending_params tp
+                    inner join aggregate_interval_plays aip
+                        on tp.track_id = aip.track_id;
+                INSERT INTO track_trending_scores
+                    (track_id, genre, type, version, time_range, score, created_at)
+                    select
+                        tp.track_id,
+                        tp.genre,
+                        :type,
+                        :version,
+                        :all_time_time_range,
+                        CASE
+                        WHEN tp.owner_follower_count < :y
+                            THEN 0
+                        ELSE (:N * ap.count + :R * tp.repost_count + :i * tp.save_count) * tp.karma
+                        END as all_time_score,
+                        now()
+                    from trending_params tp
+                    inner join aggregate_plays ap
+                        on tp.track_id = ap.play_item_id
+                    inner join tracks t
+                        on ap.play_item_id = t.track_id
+                    where -- same filtering for aggregate_interval_plays
+                        t.is_current is True AND
+                        t.is_delete is False AND
+                        t.is_unlisted is False AND
+                        t.stem_of is Null;
+            commit;
+        """
+        )
+        session.execute(
+            trending_track_query,
+            {
+                "week": T["week"],
+                "month": T["month"],
+                "N": N,
+                "F": F,
+                "O": O,
+                "R": R,
+                "i": i,
+                "q": q,
+                "y": y,
+                "type": self.trending_type.name,
+                "version": self.version.name,
+                "week_time_range": "week",
+                "month_time_range": "month",
+                "all_time_time_range": "allTime",
+            },
+        )
+        duration = time.time() - start_time
+        logger.info(
+            f"trending_tracks_strategy | Finished calculating trending scores in {duration} seconds",
+            extra={
+                "id": "trending_strategy",
+                "type": self.trending_type.name,
+                "version": self.version.name,
+                "duration": duration,
+            },
+        )
+
+    def get_score_params(self):
+        return {"xf": True, "pt": 0, "nm": 5}

discovery-provider/src/trending_strategies/EJ57D_underground_trending_tracks_strategy.py

@@ -0,0 +1,64 @@
+from datetime import datetime
+
+from dateutil.parser import parse
+from src.trending_strategies.base_trending_strategy import BaseTrendingStrategy
+from src.trending_strategies.trending_type_and_version import (
+    TrendingType,
+    TrendingVersion,
+)
+
+b = 5
+qw = 50
+hg = 1
+ie = 0.25
+pn = 0.01
+u = 30.0
+qq = 0.001
+oi = 20
+nb = 750
+
+
+class UndergroundTrendingTracksStrategyEJ57D(BaseTrendingStrategy):
+    def __init__(self):
+        super().__init__(TrendingType.UNDERGROUND_TRACKS, TrendingVersion.EJ57D)
+
+    def get_track_score(self, time_range, track):
+        # pylint: disable=W,C,R
+        mn = track["listens"]
+        c = track["windowed_repost_count"]
+        x = track["repost_count"]
+        v = track["windowed_save_count"]
+        ut = track["save_count"]
+        ll = track["created_at"]
+        bq = track["owner_follower_count"]
+        ty = track["owner_verified"]
+        kz = track["karma"]
+        xy = max
+        uk = pow
+        if bq < 3:
+            return {"score": 0, **track}
+        oj = qq if ty else 1
+        zu = 1
+        if bq >= nb:
+            zu = xy(uk(oi, 1 - ((1 / nb) * (bq - nb) + 1)), 1 / oi)
+        vb = (b * mn + qw * c + hg * v + ie * x + pn * ut + zu * bq) * kz * zu * oj
+        te = 7
+        fd = datetime.now()
+        xn = parse(ll)
+        ul = (fd - xn).days
+        rq = 1
+        if ul > te:
+            rq = xy((1.0 / u), (uk(u, (1 - ul / te))))
+        return {"score": vb * rq, **track}
+
+    def get_score_params(self):
+        return {
+            "S": 1500,
+            "r": 1500,
+            "q": 50,
+            "o": 21,
+            "f": 7,
+            "qr": 10,
+            "xf": True,
+            "pt": 0,
+        }

discovery-provider/src/trending_strategies/trending_strategy_factory.py

@@ -1,3 +1,12 @@
+from src.trending_strategies.EJ57D_trending_playlists_strategy import (
+    TrendingPlaylistsStrategyEJ57D,
+)
+from src.trending_strategies.EJ57D_trending_tracks_strategy import (
+    TrendingTracksStrategyEJ57D,
+)
+from src.trending_strategies.EJ57D_underground_trending_tracks_strategy import (
+    UndergroundTrendingTracksStrategyEJ57D,
+)
 from src.trending_strategies.ML51L_trending_playlists_strategy import (
     TrendingPlaylistsStrategyML51L,
 )
@@ -24,12 +33,15 @@ def __init__(self):
         self.strategies = {
             TrendingType.TRACKS: {
                 TrendingVersion.ML51L: TrendingTracksStrategyML51L(),
+                TrendingVersion.EJ57D: TrendingTracksStrategyEJ57D(),
             },
             TrendingType.UNDERGROUND_TRACKS: {
                 TrendingVersion.ML51L: UndergroundTrendingTracksStrategyML51L(),
+                TrendingVersion.EJ57D: UndergroundTrendingTracksStrategyEJ57D(),
             },
             TrendingType.PLAYLISTS: {
                 TrendingVersion.ML51L: TrendingPlaylistsStrategyML51L(),
+                TrendingVersion.EJ57D: TrendingPlaylistsStrategyEJ57D(),
             },
         }
 

discovery-provider/src/trending_strategies/trending_type_and_version.py

@@ -9,3 +9,4 @@ class TrendingType(Enum):
 
 class TrendingVersion(Enum):
     ML51L = "ML51L"
+    EJ57D = "EJ57D"