cron + task for updating addon slowness (bug 615391)

apps/addons/models.py

@@ -145,6 +145,9 @@ class Addon(amo.models.ModelBase):
                                               db_column='sharecount')
     last_updated = models.DateTimeField(db_index=True, null=True,
         help_text='Last time this add-on had a file/version update')
+    ts_slowness = models.FloatField(db_index=True, null=True,
+        help_text='How much slower this add-on makes browser ts tests. '
+                  'Read as {addon.ts_slowness}% slower.')
 
     inactive = models.BooleanField(default=False, db_index=True)
     trusted = models.BooleanField(default=False)

apps/perf/cron.py

@@ -0,0 +1,37 @@
+from django.db import connections
+
+import cronjobs
+import multidb
+from celery.messaging import establish_connection
+
+from amo.utils import chunked
+from . import tasks
+
+
+@cronjobs.register
+def update_perf():
+    cursor = connections[multidb.get_slave()].cursor()
+    # The baseline is where addon_id is null.
+    cursor.execute(
+        "SELECT AVG(average) FROM perf_results WHERE addon_id IS NULL")
+    baseline = cursor.fetchone()[0]
+
+    # The perf_results table is a mess right now, so pull out one row
+    # for each addon by finding the MAX(created) and then the AVG(average)
+    # since there are many rows with the same (addon, created).
+    # This scheme completely ignores app, os, and test.
+    cursor.execute("""
+        SELECT J.addon_id, AVG(average) av FROM perf_results P INNER JOIN
+            (SELECT addon_id, MAX(created) c FROM perf_results
+             GROUP BY addon_id) J
+        ON ((P.addon_id=J.addon_id) AND P.created=J.c)
+        WHERE test='ts'
+        GROUP BY P.addon_id
+        HAVING av > %s""", (baseline,))
+    # A bunch of (addon, perf_average) pairs.
+    perf = cursor.fetchall()
+    with establish_connection() as conn:
+        for chunk in chunked(perf, 25):
+            tasks.update_perf.apply_async(args=[baseline, chunk],
+                                          connection=conn)
+    cursor.close()

apps/perf/models.py

@@ -34,7 +34,7 @@ class Performance(amo.models.ModelBase):
 
     TEST_CHOICES = [('ts', 'Startup Time')]
 
-    addon = models.ForeignKey('addons.Addon')
+    addon = models.ForeignKey('addons.Addon', null=True)
     average = models.FloatField(default=0, db_index=True)
     appversion = models.ForeignKey(PerformanceAppVersions)
     osversion = models.ForeignKey(PerformanceOSVersion)

apps/perf/tasks.py

@@ -0,0 +1,16 @@
+import logging
+
+from celeryutils import task
+
+from addons.models import Addon
+
+log = logging.getLogger('z.perf.task')
+
+
+@task(rate_limit='1/s')
+def update_perf(baseline, perf, **kw):
+    log.info('[%s@%s] Updating perf' %
+             (len(perf), update_perf.rate_limit))
+    for addon, avg in perf:
+        num = (avg - baseline) / baseline
+        Addon.objects.filter(pk=addon).update(ts_slowness=100 * num)

migrations/101-perf-created-idx.sql

@@ -0,0 +1,6 @@
+CREATE INDEX addon_created_idx ON perf_results (addon_id, created);
+
+ALTER TABLE addons
+    ADD COLUMN ts_slowness FLOAT NULL;
+
+CREATE INDEX ts_slowness_idx ON addons (ts_slowness);