From 4f021d3c532f1fd5c3cb3ae25cd6e5d58852c3c0 Mon Sep 17 00:00:00 2001
From: Manish Goregaokar <manishsmail@gmail.com>
Date: Mon, 28 Nov 2016 10:15:19 -0800
Subject: [PATCH] Run filter-intermittents on buildbot

---
 etc/ci/buildbot_steps.yml        | 12 ++++++++----
 python/servo/testing_commands.py | 14 ++++++++++++--
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml
index 19d67c02f25f..4a944e5808d2 100644
--- a/etc/ci/buildbot_steps.yml
+++ b/etc/ci/buildbot_steps.yml
@@ -1,7 +1,8 @@
 mac-rel-wpt1:
   - ./mach build --release
   - ./mach test-wpt-failure
-  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
+  - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
   - ./mach build-cef --release
   - bash ./etc/ci/lockfile_changed.sh
@@ -21,7 +22,8 @@ mac-dev-unit:
 
 mac-rel-css:
   - ./mach build --release
-  - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
+  - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log
   - ./mach build-geckolib --release
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
@@ -57,12 +59,14 @@ linux-dev:
 linux-rel-wpt:
   - ./mach build --release --with-debug-assertions
   - ./mach test-wpt-failure
-  - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
+  - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
 
 linux-rel-css:
   - ./mach build --release --with-debug-assertions
-  - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
+  - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log
   - ./mach build-cef --release --with-debug-assertions
   - ./mach build-geckolib --release
   - ./mach test-stylo --release
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index dffc38c96465..1db3b4fb1bfd 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -68,6 +68,8 @@ def create_parser_wpt():
                         help="Run under chaos mode in rr until a failure is captured")
     parser.add_argument('--pref', default=[], action="append", dest="prefs",
                         help="Pass preferences to servo")
+    parser.add_argument('--always-succeed', default=False, action="store_true",
+                        help="Always yield exit code of zero")
     return parser
 
 
@@ -399,7 +401,11 @@ def test_wpt_failure(self):
              parser=create_parser_wpt)
     def test_wpt(self, **kwargs):
         self.ensure_bootstrapped()
-        return self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs)
+        ret = self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs)
+        if kwargs["always_succeed"]:
+            return 0
+        else:
+            return ret
 
     def _test_wpt(self, **kwargs):
         hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
@@ -556,7 +562,11 @@ def update_jquery(self, release, dev):
              parser=create_parser_wpt)
     def test_css(self, **kwargs):
         self.ensure_bootstrapped()
-        return self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs)
+        ret = self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs)
+        if kwargs["always_succeed"]:
+            return 0
+        else:
+            return ret
 
     def _test_css(self, **kwargs):
         run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))