diff --git a/etc/ci/performance/README.md b/etc/ci/performance/README.md index d65c79508b36..9c9986f937e9 100644 --- a/etc/ci/performance/README.md +++ b/etc/ci/performance/README.md @@ -18,10 +18,11 @@ Servo Page Load Time Test ## CI for Gecko * Install Firefox Nightly in your PATH -* Download [geckodriver](https://github.com/mozilla/geckodriver/releases) and add it to the `PATH` +* Download [geckodriver](https://github.com/mozilla/geckodriver/releases) and add it to the `PATH` (e.g. for Linux `export PATH=$PATH:path/to/geckodriver`) +* `export FIREFOX_BIN=/path/to/firefox` * `pip install selenium` * Run `python gecko_driver.py` to test -* Run `test_all.sh --gecko --submit` +* Run `test_all.sh --gecko --submit` (omit `--submit` if you don't want to submit to perfherder) # How it works @@ -30,6 +31,25 @@ Servo Page Load Time Test * Each testcase is a subtest on Perfherder, and their summary time is the geometric mean of all the subtests. * Notice that the test is different from the Talos TP5 test we run for Gecko. So you can NOT conclude that Servo is "faster" or "slower" than Gecko from this test. +# Comparing the performance before and after a patch + +* Run the test once before you apply a patch, and once after you apply it. +* `python test_differ.py output/perf-.json output/perf-.json` +* Green lines means loading time decreased, Blue lines means loading time increased. + +# Add your own test + +* Add you test case (html file) to the `page_load_test/` folder. For example we can create a `page_load_test/example/example.html` +* Add a manifest (or modify existing ones) named `page_load_test/example.manifest` +* Add the lines like this to the manifest: + +``` +http://localhost:8000/page_load_test/example/example.html +# This is a comment +# Pages got served on a local server at localhost:8000 +``` +* Modify the `MANIFEST=...` link in `test_all.sh` and point that to the new manifest file. + # Unit tests You can run all unit tests (include 3rd-party libraries) with `python -m pytest`. diff --git a/etc/ci/performance/gecko_driver.py b/etc/ci/performance/gecko_driver.py index f8772a81c6f3..fe1450d1d1c6 100644 --- a/etc/ci/performance/gecko_driver.py +++ b/etc/ci/performance/gecko_driver.py @@ -6,13 +6,22 @@ from contextlib import contextmanager import json +import os from selenium import webdriver from selenium.common.exceptions import TimeoutException +import sys @contextmanager def create_gecko_session(): - firefox_binary = "./firefox/firefox/firefox" + try: + firefox_binary = os.environ['FIREFOX_BIN'] + except KeyError: + print("+=============================================================+") + print("| You must set the path to your firefox binary to FIREFOX_BIN |") + print("+=============================================================+") + sys.exit() + driver = webdriver.Firefox(firefox_binary=firefox_binary) yield driver # driver.quit() gives an "'NoneType' object has no attribute 'path'" error. @@ -90,6 +99,7 @@ def run_gecko_test(testcase, timeout): return [timings] + if __name__ == '__main__': # Just for manual testing from pprint import pprint