Added test differ.

etc/ci/performance/test_differ.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+
+
+parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.")
+parser.add_argument("file1", help="the first output json from runner")
+parser.add_argument("file2", help="the second output json from runner")
+
+args = parser.parse_args()
+
+def load_data(filename):
+    with open(filename, 'r') as f:
+        results = {}
+        totals = {}
+        counts = {}
+        records = json.load(f)
+        for record in records:
+            key = record.get('testcase')
+            value = record.get('domComplete') - record.get('domLoading')
+            totals[key] = totals.get('key', 0) + value
+            counts[key] = counts.get('key', 0) + 1
+            results[key] = round(totals[key] / counts[key])
+        return results
+
+data1 = load_data(args.file1)
+data2 = load_data(args.file2)
+keys = set(data1.keys()).union(data2.keys())
+
+BLUE = '\033[94m'
+GREEN = '\033[92m'
+WARNING = '\033[93m'
+END = '\033[0m'
+
+for key in keys:
+    value1 = data1.get(key)
+    value2 = data2.get(key)
+    if value1 and not(value2):
+        print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END))
+    elif value2 and not(value1):
+        print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END))
+    elif value1 and value2:
+        diff = value2 - value1
+        change = diff / value1 
+        color = BLUE if value1 <= value2 else GREEN
+        print("{}Test {}: first={}, second={}, diff={}, change={:.2%}.{}".format(color, key, value1, value2, diff, change, END))