Enable logging test stability results to GitHub PR.

.travis.yml

@@ -14,6 +14,7 @@ before_install:
   - sh -e /etc/init.d/xvfb start
 install:
   - pip install -U setuptools
+  - pip install -U requests
 matrix:
   include:
     - os: linux
@@ -39,6 +40,7 @@ matrix:
             - libappindicator1
             - fonts-liberation
       env:
+        - secure: "YTSXPwI0DyCA1GhYrLT9KMEV6b7QQKuEeaQgeFDP38OTzJ1+cIj3CC4SRNqbnJ/6SJwPGcdqSxLuV8m4e5HFFnyCcQnJe6h8EMsTehZ7W3j/fP9UYrJqYqvGpe3Vj3xblO5pwBYmq7sg3jAmmuCgAgOW6VGf7cRMucrsmFeo7VM="
         - SCRIPT=ci_stability.sh
         - PRODUCT=chrome
 script:

check_stability.py

@@ -1,10 +1,14 @@
 import argparse
+import json
 import logging
 import os
 import subprocess
 import sys
+import traceback
 from collections import defaultdict
 
+import requests
+
 from wptrunner import wptrunner
 from wptrunner import wptcommandline
 from mozlog import reader
@@ -17,12 +21,57 @@ def setup_logging():
     formatter = logging.Formatter(logging.BASIC_FORMAT, None)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
-
+    logger.setLevel(logging.DEBUG)
 
 setup_logging()
 
 
+def setup_github_logging(args):
+    gh_handler = None
+    if args.comment_pr:
+        if args.gh_token:
+            try:
+                pr_number = int(args.comment_pr)
+            except ValueError:
+                pass
+            else:
+                gh_handler = GitHubCommentHandler(args.gh_token, pr_number)
+                logger.debug("Setting up GitHub logging")
+                logger.addHandler(gh_handler)
+        else:
+            logger.error("Must provide --comment-pr and --github-token together")
+    return gh_handler
+
+
+class GitHubCommentHandler(logging.Handler):
+    def __init__(self, token, pull_number):
+        logging.Handler.__init__(self)
+        self.token = token
+        self.pull_number = pull_number
+        self.log_data = []
+
+    def emit(self, record):
+        try:
+            msg = self.format(record)
+            self.log_data.append(msg)
+        except Exception:
+            self.handleError(record)
+
+    def send(self):
+        headers = {"Accept": "application/vnd.github.v3+json"}
+        auth = (self.token, "x-oauth-basic")
+        url = "https://api.github.com/repos/w3c/web-platform-tests/issues/%s/comments" %(
+            self.pull_number,)
+        resp = requests.post(
+            url,
+            data=json.dumps({"body": "\n".join(self.log_data)}),
+            headers=headers,
+            auth=auth
+        )
+        resp.raise_for_status()
+        self.log_data = []
+
+
 class Firefox(object):
     product = "firefox"
 
@@ -106,30 +155,43 @@ def err_string(results_dict):
     rv = []
     for key, value in sorted(results_dict.items()):
         rv.append("%s: %i" % (key, value))
-    return " ".join(rv)
+    rv = " ".join(rv)
+    if len(results_dict) > 1:
+        rv = "**%s**" % rv
+    return rv
 
 
-def check_consistent(log, iterations):
+def process_results(log, iterations):
     inconsistent = []
     handler = LogHandler()
     reader.handle_log(reader.read(log), handler)
     results = handler.results
     for test, test_results in results.iteritems():
-        parent = test_results.pop(None)
-        if is_inconsistent(parent, iterations):
-            inconsistent.append((test, None, parent))
-            write = logger.error
-        else:
-            write = logger.info
-        write("| %s => %s" % (test, err_string(parent)))
         for subtest, result in test_results.iteritems():
             if is_inconsistent(result, iterations):
                 inconsistent.append((test, subtest, result))
-                write = logger.error
-            else:
-                write = logger.info
-            write("| - %s => %s" % (test, err_string(result)))
-    return inconsistent
+    return results, inconsistent
+
+
+def write_inconsistent(inconsistent):
+    logger.error("## Unstable results ##\n")
+    logger.error("| Test | Subtest | Results |")
+    logger.error("|------|---------|---------|")
+    for test, subtest, results in inconsistent:
+        logger.error("%s | %s | %s" % (test,
+                                       subtest if subtest else "(parent)",
+                                       err_string(results)))
+
+
+def write_results(results, iterations):
+    logger.info("## All results ##\n")
+    logger.info("| Test | Subtest | Results |")
+    logger.info("|------|---------|---------|")
+    for test, test_results in results.iteritems():
+        parent = test_results.pop(None)
+        logger.info("| %s |  | %s |" % (test, err_string(parent)))
+        for subtest, result in test_results.iteritems():
+            logger.info("|  | %s | %s |" % (subtest, err_string(result)))
 
 
 def get_parser():
@@ -143,27 +205,38 @@ def get_parser():
                         default=10,
                         type=int,
                         help="Number of times to run tests")
+    parser.add_argument("--gh-token",
+                        action="store",
+                        help="OAuth token to use for accessing GitHub api")
+    parser.add_argument("--comment-pr",
+                        action="store",
+                        help="PR to comment on with stability results")
     parser.add_argument("browser",
                         action="store",
                         help="Browser to run against")
     return parser
 
 
 def main():
+    retcode = 0
     parser = get_parser()
     args = parser.parse_args()
 
+    gh_handler = setup_github_logging(args)
+
+    logger.info("Testing in **%s**" % args.browser.title())
+
     browser_cls = {"firefox": Firefox,
                    "chrome": Chrome}.get(args.browser)
     if browser_cls is None:
         logger.critical("Unrecognised browser %s" % args.browser)
-        sys.exit(2)
+        return 2
 
     # For now just pass the whole list of changed files to wptrunner and
     # assume that it will run everything that's actually a test
     files_changed = get_files_changed(args.root)
 
-    logger.info("Files changed:\n  %s" % "\n  ".join(files_changed))
+    logger.info("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
 
     browser = browser_cls()
     kwargs = wptrunner_args(args.root,
@@ -176,20 +249,29 @@ def main():
                                  "raw": log})
         wptrunner.run_tests(**kwargs)
 
-    logger.info("Test runs done")
-
     with open("raw.log", "rb") as log:
-        inconsistent = check_consistent(log, args.iterations)
+        results, inconsistent = process_results(log, args.iterations)
 
     if inconsistent:
-        logger.error("Got unstable results:")
-        for test, subtest, results in inconsistent:
-            logger.error("%s | %s | %s" % (test,
-                                           subtest if subtest else "(parent)",
-                                           err_string(results)))
-        sys.exit(1)
-    logger.info("All results were stable")
+        write_inconsistent(inconsistent)
+        retcode = 1
+    else:
+        logger.info("All results were stable\n")
+
+    write_results(results, args.iterations)
+
+    try:
+        if gh_handler:
+            gh_handler.send()
+    except Exception:
+        logger.error(traceback.format_exc())
+    return retcode
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        retcode = main()
+    except:
+        raise
+    else:
+        sys.exit(retcode)

ci_stability.sh

@@ -96,7 +96,7 @@ install_chromedriver() {
 
 test_stability() {
     cd $WPT_HOME
-    python check_stability.py --root $BUILD_HOME $PRODUCT
+    python check_stability.py --root $BUILD_HOME --comment-pr ${TRAVIS_PULL_REQUEST} --gh-token ${GH_TOKEN} $PRODUCT
 }
 
 main() {