Add a benchmarking module to wrench.

This adds two new commands to wrench:

perf - This runs all the benchmarks in the benchmarks manifest, and records
the timing of various profile counters into a JSON file. This is
very similar to how the reftest code works.

compare_perf - This allows comparing two files output from the perf command above.

There are two new workflows enabled by this change:

(1) Working locally:
 - Run `perf` to get a baseline benchmark.
 - Develop new feature or optimization.
 - Run `perf` to get new results, and use `compare_perf` to compare to baseline.

(2) Automated performance measurement:
 - This is not available yet, but the idea is to upload the results
   of the perf command to a web server, which graphs the results
   over time.

New benchmarks can be added to the benchmarks/ directory, in a similar
way to adding reftests.

wrench/Cargo.toml

@@ -24,6 +24,8 @@ osmesa-sys = { version = "0.1.2", optional = true }
 osmesa-src = { git = "https://github.com/servo/osmesa-src", optional = true }
 webrender = {path = "../webrender"}
 webrender_traits = {path = "../webrender_traits"}
+serde_derive = "0.9"
+serde = "0.9"
 
 [features]
 headless = [ "osmesa-sys", "osmesa-src" ]

wrench/benchmarks/benchmarks.list

@@ -0,0 +1,3 @@
+simple-batching.yaml
+large-clip-rect.yaml
+

wrench/benchmarks/large-clip-rect.yaml

@@ -0,0 +1,59 @@
+---
+root:
+  items:
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16
+        - type: rect
+          bounds: [0, 0, 1024, 1024]
+          color: red
+          clip:
+            complex:
+              - rect: [0, 0, 1024, 1024]
+                radius: 16

wrench/benchmarks/simple-batching.yaml

@@ -0,0 +1,45 @@
+---
+root:
+  items:
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green
+        - type: rect
+          bounds: [0, 0, 512, 512]
+          color: green

wrench/src/args.yaml

@@ -103,3 +103,21 @@ subcommands:
               help: a specific reftest or directory to run
               required: false
               index: 1
+    - perf:
+        about: run benchmarks
+        args:
+          - filename:
+              help: name of the file to save benchmarks to
+              required: true
+              index: 1
+    - compare_perf:
+        about: compare two benchmark files
+        args:
+          - first_filename:
+              help: first benchmark file to compare
+              required: true
+              index: 1
+          - second_filename:
+              help: second benchmark file to compare
+              required: true
+              index: 2

wrench/src/main.rs

@@ -21,6 +21,8 @@ extern crate image;
 extern crate lazy_static;
 #[cfg(feature = "headless")]
 extern crate osmesa_sys;
+#[macro_use]
+extern crate serde_derive;
 extern crate serde_json;
 extern crate time;
 extern crate webrender;
@@ -30,6 +32,7 @@ extern crate yaml_rust;
 mod binary_frame_reader;
 mod json_frame_writer;
 mod parse_function;
+mod perf;
 mod png;
 mod reftest;
 mod scene;
@@ -43,6 +46,7 @@ use gleam::gl;
 use glutin::{ElementState, VirtualKeyCode, WindowProxy};
 use image::ColorType;
 use image::png::PNGEncoder;
+use perf::PerfHarness;
 use reftest::ReftestHarness;
 use std::cmp::{max, min};
 #[cfg(feature = "headless")]
@@ -324,6 +328,17 @@ fn main() {
             let specific_reftest = subargs.value_of("REFTEST").map(|x| Path::new(x));
             harness.run(&base_manifest, specific_reftest);
             return;
+        } else if let Some(subargs) = args.subcommand_matches("perf") {
+            let harness = PerfHarness::new(&mut wrench, &mut window);
+            let base_manifest = Path::new("benchmarks/benchmarks.list");
+            let filename = subargs.value_of("filename").unwrap();
+            harness.run(&base_manifest, filename);
+            return;
+        } else if let Some(subargs) = args.subcommand_matches("compare_perf") {
+            let first_filename = subargs.value_of("first_filename").unwrap();
+            let second_filename = subargs.value_of("second_filename").unwrap();
+            perf::compare(first_filename, second_filename);
+            return;
         } else {
             panic!("Should never have gotten here");
         };