Etc ci performance run locally

etc/ci/performance/page_load_test/tp5n/20160509.manifest

@@ -1,52 +1,52 @@
-http://localhost:8000/tp6/news.ycombinator.com/index.html
-# http://localhost:8000/page_load_test/163.com/www.163.com/index.html
-http://localhost:8000/page_load_test/56.com/www.56.com/index.html
-# http://localhost:8000/page_load_test/aljazeera.net/aljazeera.net/portal.html
-http://localhost:8000/page_load_test/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html
-http://localhost:8000/page_load_test/bbc.co.uk/www.bbc.co.uk/news/index.html
-http://localhost:8000/page_load_test/beatonna.livejournal.com/beatonna.livejournal.com/index.html
-# http://localhost:8000/page_load_test/bild.de/www.bild.de/index.html
-http://localhost:8000/page_load_test/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html
-http://localhost:8000/page_load_test/chemistry.about.com/chemistry.about.com/index.html
-# http://localhost:8000/page_load_test/chinaz.com/chinaz.com/index.html
-http://localhost:8000/page_load_test/cnn.com/www.cnn.com/index.html
-http://localhost:8000/page_load_test/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html
-http://localhost:8000/page_load_test/dailymotion.com/www.dailymotion.com/us.html
-# http://localhost:8000/page_load_test/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html
-http://localhost:8000/page_load_test/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html
-http://localhost:8000/page_load_test/globo.com/www.globo.com/index.html
-http://localhost:8000/page_load_test/google.com/www.google.com/search@q=mozilla.html
-http://localhost:8000/page_load_test/goo.ne.jp/goo.ne.jp/index.html
-# http://localhost:8000/page_load_test/guardian.co.uk/www.guardian.co.uk/index.html
-# http://localhost:8000/page_load_test/homeway.com.cn/www.hexun.com/index.html
-http://localhost:8000/page_load_test/huffingtonpost.com/www.huffingtonpost.com/index.html
-# http://localhost:8000/page_load_test/ifeng.com/ifeng.com/index.html
-# http://localhost:8000/page_load_test/imdb.com/www.imdb.com/title/tt1099212/index.html
-http://localhost:8000/page_load_test/imgur.com/imgur.com/gallery/index.html
-# http://localhost:8000/page_load_test/indiatimes.com/www.indiatimes.com/index.html
-http://localhost:8000/page_load_test/mail.ru/mail.ru/index.html
-# http://localhost:8000/page_load_test/mashable.com/mashable.com/index.html
-http://localhost:8000/page_load_test/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html
-http://localhost:8000/page_load_test/myspace.com/www.myspace.com/albumart.html
-# http://localhost:8000/page_load_test/naver.com/www.naver.com/index.html
-# http://localhost:8000/page_load_test/noimpactman.typepad.com/noimpactman.typepad.com/index.html
-http://localhost:8000/page_load_test/page.renren.com/page.renren.com/index.html
-# http://localhost:8000/page_load_test/people.com.cn/people.com.cn/index.html
-http://localhost:8000/page_load_test/rakuten.co.jp/www.rakuten.co.jp/index.html
-http://localhost:8000/page_load_test/reddit.com/www.reddit.com/index.html
-# http://localhost:8000/page_load_test/reuters.com/www.reuters.com/index.html
-# http://localhost:8000/page_load_test/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html
-# http://localhost:8000/page_load_test/sohu.com/www.sohu.com/index.html
-http://localhost:8000/page_load_test/spiegel.de/www.spiegel.de/index.html
-http://localhost:8000/page_load_test/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html
-# http://localhost:8000/page_load_test/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html
-# http://localhost:8000/page_load_test/thepiratebay.org/thepiratebay.org/top/201.html
-http://localhost:8000/page_load_test/tudou.com/www.tudou.com/index.html
-# http://localhost:8000/page_load_test/uol.com.br/www.uol.com.br/index.html
-http://localhost:8000/page_load_test/w3.org/www.w3.org/standards/webdesign/htmlcss.html
-# http://localhost:8000/page_load_test/wsj.com/online.wsj.com/home-page.html
-# http://localhost:8000/page_load_test/xinhuanet.com/xinhuanet.com/index.html
-http://localhost:8000/page_load_test/xunlei.com/xunlei.com/index.html
-http://localhost:8000/page_load_test/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html
-http://localhost:8000/page_load_test/youku.com/www.youku.com/index.html
-http://localhost:8000/page_load_test/youtube.com/www.youtube.com/music.html
+# http://localhost:8000/tp6/news.ycombinator.com/index.html
+# http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html
+http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html
+# http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
+http://localhost:8000/page_load_test/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html
+http://localhost:8000/page_load_test/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html
+http://localhost:8000/page_load_test/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html
+# http://localhost:8000/page_load_test/tp5n/bild.de/www.bild.de/index.html
+http://localhost:8000/page_load_test/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html
+http://localhost:8000/page_load_test/tp5n/chemistry.about.com/chemistry.about.com/index.html
+# http://localhost:8000/page_load_test/tp5n/chinaz.com/chinaz.com/index.html
+http://localhost:8000/page_load_test/tp5n/cnn.com/www.cnn.com/index.html
+http://localhost:8000/page_load_test/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html
+http://localhost:8000/page_load_test/tp5n/dailymotion.com/www.dailymotion.com/us.html
+# http://localhost:8000/page_load_test/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html
+http://localhost:8000/page_load_test/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html
+http://localhost:8000/page_load_test/tp5n/globo.com/www.globo.com/index.html
+http://localhost:8000/page_load_test/tp5n/google.com/www.google.com/search@q=mozilla.html
+http://localhost:8000/page_load_test/tp5n/goo.ne.jp/goo.ne.jp/index.html
+# http://localhost:8000/page_load_test/tp5n/guardian.co.uk/www.guardian.co.uk/index.html
+# http://localhost:8000/page_load_test/tp5n/homeway.com.cn/www.hexun.com/index.html
+http://localhost:8000/page_load_test/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html
+# http://localhost:8000/page_load_test/tp5n/ifeng.com/ifeng.com/index.html
+# http://localhost:8000/page_load_test/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html
+http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html
+# http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html
+http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html
+# http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html
+# http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html
+http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html
+# http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html
+# http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html
+http://localhost:8000/page_load_test/tp5n/page.renren.com/page.renren.com/index.html
+# http://localhost:8000/page_load_test/tp5n/people.com.cn/people.com.cn/index.html
+http://localhost:8000/page_load_test/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html
+http://localhost:8000/page_load_test/tp5n/reddit.com/www.reddit.com/index.html
+# http://localhost:8000/page_load_test/tp5n/reuters.com/www.reuters.com/index.html
+# http://localhost:8000/page_load_test/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html
+# http://localhost:8000/page_load_test/tp5n/sohu.com/www.sohu.com/index.html
+http://localhost:8000/page_load_test/tp5n/spiegel.de/www.spiegel.de/index.html
+http://localhost:8000/page_load_test/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html
+# http://localhost:8000/page_load_test/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html
+# http://localhost:8000/page_load_test/tp5n/thepiratebay.org/thepiratebay.org/top/201.html
+http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html
+# http://localhost:8000/page_load_test/tp5n/uol.com.br/www.uol.com.br/index.html
+http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html
+# http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html
+# http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html
+# http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html
+http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html
+http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html
+http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html

etc/ci/performance/runner.py

@@ -28,8 +28,7 @@ def execute_test(url, command, timeout):
     print(' '.join(command))
     print("Timeout:{}".format(timeout))
     try:
-        return subprocess.check_output(command, stderr=subprocess.STDOUT,
-                                       shell=True, timeout=timeout)
+        return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
     except subprocess.CalledProcessError as e:
         print("Unexpected Fail:")
         print(e)
@@ -43,7 +42,7 @@ def execute_test(url, command, timeout):
 def get_servo_command(url):
     ua_script_path = "{}/user-agent-js".format(os.getcwd())
     return ["./servo/servo", url,
-            " --userscripts", ua_script_path,
+            "--userscripts", ua_script_path,
             "-x", "-o", "output.png"]
 
 
@@ -67,7 +66,7 @@ def parse_log(log, testcase=None):
             copy = False
             blocks.append(block)
             block = []
-        elif copy:
+        elif copy and line.strip().startswith("[PERF]"):
             block.append(line)
 
     # We need to still include the failed tests, otherwise Treeherder will
@@ -113,21 +112,19 @@ def parse_block(block):
                 print('-----')
                 return placeholder
 
-            if key == "testcase":
+            if key == "testcase" or key == "title":
                 timing[key] = value
             else:
                 timing[key] = None if (value == "undefined") else int(value)
 
-        if testcase is not None and timing['testcase'] != testcase:
-            print('[DEBUG] log:')
-            print('-----')
-            print(log)
-            print('-----')
-            return placeholder
-
         return timing
 
-    if len(blocks) == 0:
+    def valid_timing(timing):
+        return (timing.get('title') != 'Error response') and (testcase is None or timing.get('testcase') == testcase)
+
+    timings = list(filter(valid_timing, map(parse_block, blocks)))
+
+    if len(timings) == 0:
         print("Didn't find any perf data in the log, test timeout?")
         print("Fillng in a dummy perf data")
         print('[DEBUG] log:')
@@ -137,7 +134,7 @@ def parse_block(block):
 
         return [placeholder]
     else:
-        return map(parse_block, blocks)
+        return timings
 
 
 def filter_result_by_manifest(result_json, manifest):
@@ -242,7 +239,7 @@ def main():
         testcases = load_manifest(args.tp5_manifest)
         results = []
         for testcase in testcases:
-            command = (["timeout", "{timeout}s".format(args.timeout)] +
+            command = (["timeout", "{}s".format(args.timeout)] +
                        command_factory(testcase))
             for run in range(args.runs):
                 print("Running test {}/{} on {}".format(run + 1,

etc/ci/performance/test_all.sh

@@ -8,19 +8,29 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+while (( "${#}" ))
+do
 case "${1}" in
   --servo)
     engine="--engine servo"
     ;;
   --gecko)
     engine="--engine gecko"
     ;;
+  --submit)
+    submit=1
+    ;;
   *)
-    # This branch should never be reached with set -o nounset
-    echo "You didn't specify the engine to run."
+    echo "Unknown option ${1}."
     exit
     ;;
 esac
+shift
+done
+
+if [[ -z "${engine:-}" ]];
+then echo "You didn't specify the engine to run: --servo or --gecko."; exit;
+fi
 
 echo "Starting the local server"
 python3 -m http.server > /dev/null 2>&1 &
@@ -33,15 +43,19 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes
 PERF_FILE="output/perf-$(date --iso-8601=seconds).json"
 
 echo "Running tests"
-python3 runner.py "${engine}" --runs 3 "${MANIFEST}" "${PERF_FILE}"
-
-echo "Submitting to Perfherder"
-# Perfherder SSL check will fail if time is not accurate,
-# sync time before you submit
-# TODO: we are using Servo's revision hash for Gecko's result to make both
-# results appear on the same date. Use the correct result when Perfherder
-# allows us to change the date.
-python3 submit_to_perfherder.py "${engine}" "${PERF_FILE}" servo/revision.json
-
-# Kill the http server
+python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}"
+
+if [[ "${submit:-}" ]];
+then
+    echo "Submitting to Perfherder"
+    # Perfherder SSL check will fail if time is not accurate,
+    # sync time before you submit
+    # TODO: we are using Servo's revision hash for Gecko's result to make both
+    # results appear on the same date. Use the correct result when Perfherder
+    # allows us to change the date.
+    python3 submit_to_perfherder.py \
+            "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json
+fi
+
+echo "Stopping the local server"
 trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT

etc/ci/performance/test_differ.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+
+
+parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.")
+parser.add_argument("file1", help="the first output json from runner")
+parser.add_argument("file2", help="the second output json from runner")
+
+args = parser.parse_args()
+
+
+def load_data(filename):
+    with open(filename, 'r') as f:
+        results = {}
+        totals = {}
+        counts = {}
+        records = json.load(f)
+        for record in records:
+            key = record.get('testcase')
+            value = record.get('domComplete') - record.get('domLoading')
+            totals[key] = totals.get('key', 0) + value
+            counts[key] = counts.get('key', 0) + 1
+            results[key] = round(totals[key] / counts[key])
+        return results
+
+data1 = load_data(args.file1)
+data2 = load_data(args.file2)
+keys = set(data1.keys()).union(data2.keys())
+
+BLUE = '\033[94m'
+GREEN = '\033[92m'
+WARNING = '\033[93m'
+END = '\033[0m'
+
+for key in keys:
+    value1 = data1.get(key)
+    value2 = data2.get(key)
+    if value1 and not(value2):
+        print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END))
+    elif value2 and not(value1):
+        print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END))
+    elif value1 and value2:
+        diff = value2 - value1
+        change = diff / value1
+        color = BLUE if value1 <= value2 else GREEN
+        print("{}{:6} {:6} {:+6} {:+8.2%}   {}.{}".format(color, value1, value2, diff, change, key, END))

etc/ci/performance/user-agent-js/01.perf-timing.js

@@ -12,6 +12,7 @@ function formatLine(name, t) {
 function printPerfTiming() {
   print("[PERF] perf block start")
   formatLine("testcase", window.location);
+  formatLine("title", document.title.replace(/,/g, ","));
   formatLine("navigationStart", performance.timing.navigationStart);
   formatLine("unloadEventStart", performance.timing.unloadEventStart);
   formatLine("unloadEventEnd", performance.timing.unloadEventEnd);