EditCounter: Rework avg/small/larges and compute only against past 5000 edits

app/Resources/views/editCounter/general_stats.html.twig

@@ -89,7 +89,7 @@
         <tr><td>{{ msg('last-month') }}</td><td>{{ ec.countRevisionsInLast('month')|number_format }}</td></tr>
         <tr><td>{{ msg('last-year') }}</td><td>{{ ec.countRevisionsInLast('year')|number_format }}</td></tr>
         <tr>
-            <td>{{ msg("average-edits-per-day", [msg("days", [1])]) }}</td>
+            <td>{{ msg('average-edits-per-day', [msg('days', [1])]) }}</td>
             <td>
                 {{ ec.averageRevisionsPerDay|number_format(1) }}
                 <span class="text-muted">
@@ -98,8 +98,8 @@
             </td>
         </tr>
         <tr>
-            <td>{{ msg('average-edit-size') }}</td>
-            <td>{{ ec.averageRevisionSize|number_format(1) }}</td>
+            <td>{{ msg('average-edit-size') }}*</td>
+            <td>{{ ec.averageEditSize|number_format(1) }} {{ msg('num-bytes', [ec.averageEditSize]) }}</td>
         </tr>
     </tbody></table>
 </div>
@@ -202,19 +202,19 @@
                 </td>
             </tr>
             <tr>
-                <td>{{ msg('small-edits') }}</td>
+                <td>{{ msg('small-edits') }}*</td>
                 <td>
-                    {{ ec.countSmallRevisions|number_format }}
+                    {{ ec.countSmallEdits|number_format }}
                     &middot;
-                    ({{ ((ec.countSmallRevisions / ec.countLiveRevisions) * 100)|percent_format }})
+                    ({{ ((ec.countSmallEdits / ec.countLast5000) * 100)|percent_format }})
                 </td>
             </tr>
             <tr>
-                <td>{{ msg('large-edits') }}</td>
+                <td>{{ msg('large-edits') }}*</td>
                 <td>
-                    {{ ec.countLargeRevisions|number_format }}
+                    {{ ec.countLargeEdits|number_format }}
                     &middot;
-                    ({{ ((ec.countLargeRevisions / ec.countLiveRevisions) * 100)|percent_format }})
+                    ({{ ((ec.countLargeEdits / ec.countLast5000) * 100)|percent_format }})
                 </td>
             </tr>
         </tbody>
@@ -423,31 +423,34 @@
     {{
         chart.pie_chart('small_edits',
             [{
-                label: '< 20 ' ~ msg('num-bytes', [20]),
-                value: ec.countSmallRevisions,
-                percentage: ((ec.countSmallRevisions / ec.countLiveRevisions) * 100)
+                label: '< 20 ' ~ msg('num-bytes', [20]) ~ '*',
+                value: ec.countSmallEdits,
+                percentage: ((ec.countSmallEdits / ec.countLast5000) * 100)
             },
             {
-                label: '≥ 20 ' ~ msg('num-bytes', [20]),
-                value: ec.countLiveRevisions - ec.countSmallRevisions,
-                percentage: 100 - ((ec.countSmallRevisions / ec.countLiveRevisions) * 100)
+                label: '≥ 20 ' ~ msg('num-bytes', [20]) ~ '*',
+                value: ec.countLast5000 - ec.countSmallEdits,
+                percentage: 100 - ((ec.countSmallEdits / ec.countLast5000) * 100)
             }]
         )
     }}
     {{
         chart.pie_chart('large_edits',
             [{
-                label: '< 1000 ' ~ msg('num-bytes', [1000]),
-                value: ec.countLargeRevisions,
-                percentage: ((ec.countLargeRevisions / ec.countLiveRevisions) * 100)
+                label: '< 1000 ' ~ msg('num-bytes', [1000]) ~ '*',
+                value: ec.countLargeEdits,
+                percentage: ((ec.countLargeEdits / ec.countLast5000) * 100)
             },
             {
-                label: '≥ 1000 ' ~ msg('num-bytes', [1000]),
-                value: ec.countLiveRevisions - ec.countLargeRevisions,
-                percentage: 100 - ((ec.countLargeRevisions / ec.countLiveRevisions) * 100)
+                label: '≥ 1000 ' ~ msg('num-bytes', [1000]) ~ '*',
+                value: ec.countLast5000 - ec.countLargeEdits,
+                percentage: 100 - ((ec.countLargeEdits / ec.countLast5000) * 100)
             }]
         )
     }}
+    <div class="footnotes">
+        * {{ msg('data-limit', [5000, 5000|number_format]) }}
+    </div>
 </div>
 
 {% if not is_sub_request %}

i18n/en.json

@@ -55,6 +55,7 @@
   "created-by": "Created by",
   "current-admins": "Current admins",
   "current-size": "Current size",
+  "data-limit": "Data limited to the past $2 {{PLURAL:$1|edit|edits}}",
   "date": "Date",
   "delete": "Delete",
   "deleted": "Deleted",

i18n/qqq.json

@@ -54,6 +54,7 @@
   "created-by": "Created by",
   "current-admins": "Label for the current number of admins for a project.",
   "current-size": "Indicates the current size of a page.\n{{Identical|Current size}}",
+  "data-limit": "Footnote indictaing that data only accounts for the past $1 edits the user has made. $1 is the raw number that can be used in {{PLURAL}}, and $2 is the formatted number.",
   "date": "Label for a date field\n{{Identical|Date}}",
   "delete": "Name of a MediaWiki log action, used as header of a row in the table of log action counts for a user.\n{{Identical|Delete}}",
   "deleted": "Deleted in the context of deleted text or deleted edits.\n{{Identical|Deleted}}",

src/Xtools/EditCounter.php

@@ -37,6 +37,12 @@ class EditCounter extends Model
     /** @var array Block data, with keys 'set' and 'received'. */
     protected $blocks;
 
+    /**
+     * Revision size data, with keys 'average_size', 'large_edits' and 'small_edits'.
+     * @var string[] As returned by the DB, unconverted to int or float
+     */
+    protected $editSizeData;
+
     /**
      * Duration of the longest block in days; -1 if indefinite,
      *   or false if could not be parsed from log params
@@ -155,37 +161,6 @@ public function countMinorRevisions()
         return isset($revCounts['minor']) ? $revCounts['minor'] : 0;
     }
 
-    /**
-     * Get the total number of revisions under 20 bytes.
-     */
-    public function countSmallRevisions()
-    {
-        $revCounts = $this->getPairData();
-        return isset($revCounts['small']) ? $revCounts['small'] : 0;
-    }
-
-    /**
-     * Get the total number of revisions over 1000 bytes.
-     */
-    public function countLargeRevisions()
-    {
-        $revCounts = $this->getPairData();
-        return isset($revCounts['large']) ? $revCounts['large'] : 0;
-    }
-
-    /**
-     * Get the average revision size for the user.
-     * @return float Size in bytes.
-     */
-    public function averageRevisionSize()
-    {
-        $revisionCounts = $this->getPairData();
-        if (!isset($revisionCounts['average_size'])) {
-            return 0;
-        }
-        return round($revisionCounts['average_size'], 3);
-    }
-
     /**
      * Get the total number of non-deleted pages edited by the user.
      * @return int
@@ -788,4 +763,61 @@ public function globalEdits($max)
         $globalEdits = array_slice($globalEdits, 0, $max);
         return $globalEdits;
     }
+
+    /**
+     * Get average edit size, and number of large and small edits.
+     * @return int[]
+     */
+    protected function getEditSizeData()
+    {
+        if (! is_array($this->editSizeData)) {
+            $this->editSizeData = $this->getRepository()
+                ->getEditSizeData($this->project, $this->user);
+        }
+        return $this->editSizeData;
+    }
+
+    /**
+     * Get the total edit count of this user or 5,000 if they've made more than 5,000 edits.
+     * This is used to ensure percentages of small and large edits are computed properly.
+     * @return int
+     */
+    public function countLast5000()
+    {
+        return $this->countLiveRevisions() > 5000 ? 5000 : $this->countLiveRevisions();
+    }
+
+    /**
+     * Get the number of edits under 20 bytes of the user's past 5000 edits.
+     * @return int
+     */
+    public function countSmallEdits()
+    {
+        $editSizeData = $this->getEditSizeData();
+        return isset($editSizeData['small_edits']) ? (int) $editSizeData['small_edits'] : 0;
+    }
+
+    /**
+     * Get the total number of edits over 1000 bytes of the user's past 5000 edits.
+     * @return int
+     */
+    public function countLargeEdits()
+    {
+        $editSizeData = $this->getEditSizeData();
+        return isset($editSizeData['large_edits']) ? (int) $editSizeData['large_edits'] : 0;
+    }
+
+    /**
+     * Get the average size of the user's past 5000 edits.
+     * @return float Size in bytes.
+     */
+    public function averageEditSize()
+    {
+        $editSizeData = $this->getEditSizeData();
+        if (isset($editSizeData['average_size'])) {
+            return round($editSizeData['average_size'], 3);
+        } else {
+            return 0;
+        }
+    }
 }

src/Xtools/EditCounterRepository.php

@@ -56,20 +56,11 @@ public function getPairData(Project $project, User $user)
             SELECT 'year' AS `key`, COUNT(rev_id) AS val FROM $revisionTable
                 WHERE rev_user = :userId AND rev_timestamp >= DATE_SUB(NOW(), INTERVAL 1 YEAR)
             ) UNION (
-            SELECT 'small' AS `key`, COUNT(rev_id) AS val FROM $revisionTable
-                WHERE rev_user = :userId AND rev_len < 20
-            ) UNION (
-            SELECT 'large' AS `key`, COUNT(rev_id) AS val FROM $revisionTable
-                WHERE rev_user = :userId AND rev_len > 1000
-            ) UNION (
             SELECT 'with_comments' AS `key`, COUNT(rev_id) AS val FROM $revisionTable
                 WHERE rev_user = :userId AND rev_comment != ''
             ) UNION (
             SELECT 'minor' AS `key`, COUNT(rev_id) AS val FROM $revisionTable
                 WHERE rev_user = :userId AND rev_minor_edit = 1
-            ) UNION (
-            SELECT 'average_size' AS `key`, AVG(rev_len) AS val FROM $revisionTable
-                WHERE rev_user = :userId
 
             -- Dates.
             ) UNION (
@@ -577,7 +568,7 @@ public function getTimeCard(Project $project, User $user)
      * @param User $user The user.
      * @return integer[] Array of edit counts, keyed by all tool names from
      * app/config/semi_automated.yml
-     * @TODO Load from AutoEditsController via AJAX
+     * @todo Load from AutoEditsController via AJAX
      */
     public function countAutomatedRevisions(Project $project, User $user)
     {
@@ -620,4 +611,50 @@ public function countAutomatedRevisions(Project $project, User $user)
         $this->stopwatch->stop($cacheKey);
         return $editCounts;
     }
+
+    /**
+     * Get various data about edit sizes of the past 5,000 edits.
+     * Will cache the result for 10 minutes.
+     * @param Project $project The project.
+     * @param User $user The user.
+     * @return string[] Values with for keys 'average_size',
+     *                  'small_edits' and 'large_edits'
+     */
+    public function getEditSizeData(Project $project, User $user)
+    {
+        // Set up cache.
+        $cacheKey = 'editsizedata.'.$project->getDatabaseName().'.'.$user->getCacheKey();
+        $this->stopwatch->start($cacheKey, 'XTools');
+        if ($this->cache->hasItem($cacheKey)) {
+            return $this->cache->getItem($cacheKey)->get();
+        }
+
+        // Prepare the queries and execute them.
+        $revisionTable = $this->getTableName($project->getDatabaseName(), 'revision');
+        $userId = $user->getId($project);
+        $sql = "SELECT AVG(sizes.size) AS average_size,
+                COUNT(CASE WHEN sizes.size < 20 THEN 1 END) AS small_edits,
+                COUNT(CASE WHEN sizes.size > 1000 THEN 1 END) AS large_edits
+                FROM (
+                    SELECT (CAST(revs.rev_len AS SIGNED) - IFNULL(parentrevs.rev_len, 0)) AS size
+                    FROM $revisionTable AS revs
+                    LEFT JOIN $revisionTable AS parentrevs ON (revs.rev_parent_id = parentrevs.rev_id)
+                    WHERE revs.rev_user = :userId
+                    ORDER BY revs.rev_timestamp DESC
+                    LIMIT 5000
+                ) sizes";
+        $resultQuery = $this->getProjectsConnection()->prepare($sql);
+        $resultQuery->bindParam('userId', $userId);
+        $resultQuery->execute();
+        $results = $resultQuery->fetchAll()[0];
+
+        // Cache for 10 minutes.
+        $cacheItem = $this->cache->getItem($cacheKey);
+        $cacheItem->set($results);
+        $cacheItem->expiresAfter(new DateInterval('PT10M'));
+        $this->cache->save($cacheItem);
+
+        $this->stopwatch->stop($cacheKey);
+        return $results;
+    }
 }

web/static/css/editcounter.scss

@@ -42,4 +42,9 @@
             }
         }
     }
+
+    .footnotes {
+        clear: both;
+        padding-top: 20px;
+    }
 }