LB-221: Only calculate stats of users who haven't been done recently

listenbrainz/db/tests/test_user.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
-import time
 
+import listenbrainz.db.user as db_user
 import sqlalchemy
+import time
+import ujson
 
-import listenbrainz.db.user as db_user
 from listenbrainz import db
 from listenbrainz.db.testing import DatabaseTestCase
 
@@ -71,12 +72,11 @@ def test_increase_latest_import(self):
         user = db_user.get_by_mb_id(user['musicbrainz_id'])
         self.assertEqual(val, int(user['latest_import'].strftime('%s')))
 
-    def test_get_recently_logged_in_users(self):
-        """Tests getting recently logged in users"""
+    def test_get_users_with_uncalculated_stats(self):
 
         # create two users, set one's last_login
         # to a very old value and one's last_login
-        # to now and then call get_recently_logged_in_users
+        # to now and then call the function
         user1 = db_user.get_or_create('recentuser1')
         with db.engine.connect() as connection:
             connection.execute(sqlalchemy.text("""
@@ -97,9 +97,28 @@ def test_get_recently_logged_in_users(self):
                     'musicbrainz_id': 'recentuser2'
                 })
 
-        recent_users = db_user.get_recently_logged_in_users()
-        self.assertEqual(len(recent_users), 1)
-        self.assertEqual(recent_users[0]['musicbrainz_id'], 'recentuser2')
+        users_with_uncalculated_stats = db_user.get_users_with_uncalculated_stats()
+        self.assertEqual(len(users_with_uncalculated_stats), 1)
+        self.assertEqual(users_with_uncalculated_stats[0]['musicbrainz_id'], 'recentuser2')
+
+
+        # now if we've calculated the stats for user2 recently (just now)
+        # then the function shouldn't return user2
+
+        # put some data in the stats table for user2
+        with db.engine.connect() as connection:
+            connection.execute(sqlalchemy.text("""
+                INSERT INTO statistics.user (user_id, artist, release, recording, last_updated)
+                     VALUES (:user_id, :artist, :release, :recording, NOW())
+            """), {
+                'user_id': user2['id'],
+                'artist': ujson.dumps({}),
+                'release': ujson.dumps({}),
+                'recording': ujson.dumps({}),
+            })
+
+        users_with_uncalculated_stats = db_user.get_users_with_uncalculated_stats()
+        self.assertListEqual(users_with_uncalculated_stats, [])
 
     def test_reset_latest_import(self):
         user = db_user.get_or_create('resetlatestimportuser')

listenbrainz/db/user.py

@@ -232,16 +232,19 @@ def reset_latest_import(musicbrainz_id):
     update_latest_import(musicbrainz_id, 0)
 
 
-def get_recently_logged_in_users():
-    """Returns a list of users who have logged-in in the
-    last config.STATS_CALCULATION_LOGIN_TIME days
-    """
+def get_users_with_uncalculated_stats():
+
     with db.engine.connect() as connection:
         result = connection.execute(sqlalchemy.text("""
-                SELECT {columns}
-                  FROM "user"
-                 WHERE last_login >= NOW() - INTERVAL ':x days'
+                SELECT public."user".musicbrainz_id
+                  FROM public."user"
+             LEFT JOIN statistics.user
+                    ON public."user".id = statistics.user.user_id
+                 WHERE public."user".last_login >= NOW() - INTERVAL ':x days'
+                   AND (statistics.user.last_updated IS NULL OR statistics.user.last_updated < NOW() - INTERVAL ':y days')
                 """.format(columns=','.join(USER_GET_COLUMNS))), {
-                    'x': config.STATS_CALCULATION_LOGIN_TIME
+                    'x': config.STATS_CALCULATION_LOGIN_TIME,
+                    'y': config.STATS_CALCULATION_INTERVAL,
                 })
+
         return [dict(row) for row in result]

listenbrainz/stats/calculate.py

@@ -4,13 +4,16 @@
 import listenbrainz.stats.user as stats_user
 import time
 
-from listenbrainz import db
 from listenbrainz import config
+from listenbrainz import db
 from listenbrainz import stats
 
 
 def calculate_user_stats():
-    for user in db_user.get_recently_logged_in_users():
+    """Get the users we need to calculate our statistics for and calculate their stats.
+    """
+
+    for user in db_user.get_users_with_uncalculated_stats():
         recordings = stats_user.get_top_recordings(musicbrainz_id=user['musicbrainz_id'])
         artists = stats_user.get_top_artists(musicbrainz_id=user['musicbrainz_id'])
         releases = stats_user.get_top_releases(musicbrainz_id=user['musicbrainz_id'])
@@ -24,6 +27,7 @@ def calculate_user_stats():
             artist_count=artist_count
         )
 
+
 def calculate_stats():
     calculate_user_stats()