Enable logging test stability results to GitHub PR.

.travis.yml

@@ -14,6 +14,7 @@ before_install:
   - sh -e /etc/init.d/xvfb start
 install:
   - pip install -U setuptools
+  - pip install -U requests
 matrix:
   include:
     - os: linux
@@ -39,6 +40,7 @@ matrix:
             - libappindicator1
             - fonts-liberation
       env:
+        - secure: "YTSXPwI0DyCA1GhYrLT9KMEV6b7QQKuEeaQgeFDP38OTzJ1+cIj3CC4SRNqbnJ/6SJwPGcdqSxLuV8m4e5HFFnyCcQnJe6h8EMsTehZ7W3j/fP9UYrJqYqvGpe3Vj3xblO5pwBYmq7sg3jAmmuCgAgOW6VGf7cRMucrsmFeo7VM="
         - SCRIPT=ci_stability.sh
         - PRODUCT=chrome
 script:

check_stability.py

@@ -1,10 +1,14 @@
 import argparse
+import json
 import logging
 import os
 import subprocess
 import sys
+import traceback
 from collections import defaultdict
 
+import requests
+
 from wptrunner import wptrunner
 from wptrunner import wptcommandline
 from mozlog import reader
@@ -17,12 +21,57 @@ def setup_logging():
     formatter = logging.Formatter(logging.BASIC_FORMAT, None)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
-
+    logger.setLevel(logging.DEBUG)
 
 setup_logging()
 
 
+def setup_github_logging(args):
+    gh_handler = None
+    if args.comment_pr:
+        if args.gh_token:
+            try:
+                pr_number = int(args.comment_pr)
+            except ValueError:
+                pass
+            else:
+                gh_handler = GitHubCommentHandler(args.gh_token, pr_number)
+                logger.debug("Setting up GitHub logging")
+                logger.addHandler(gh_handler)
+        else:
+            logger.error("Must provide --comment-pr and --github-token together")
+    return gh_handler
+
+
+class GitHubCommentHandler(logging.Handler):
+    def __init__(self, token, pull_number):
+        logging.Handler.__init__(self)
+        self.token = token
+        self.pull_number = pull_number
+        self.log_data = []
+
+    def emit(self, record):
+        try:
+            msg = self.format(record)
+            self.log_data.append(msg)
+        except Exception:
+            self.handleError(record)
+
+    def send(self):
+        headers = {"Accept": "application/vnd.github.v3+json"}
+        auth = (self.token, "x-oauth-basic")
+        url = "https://api.github.com/repos/w3c/web-platform-tests/issues/%s/comments" %(
+            self.pull_number,)
+        resp = requests.post(
+            url,
+            data=json.dumps({"body": "".join(self.log_data)}),
+            headers=headers,
+            auth=auth
+        )
+        resp.raise_for_status()
+        self.log_data = []
+
+
 class Firefox(object):
     product = "firefox"
 
@@ -143,21 +192,32 @@ def get_parser():
                         default=10,
                         type=int,
                         help="Number of times to run tests")
+    parser.add_argument("--gh-token",
+                        action="store",
+                        help="OAuth token to use for accessing GitHub api")
+    parser.add_argument("--comment-pr",
+                        action="store",
+                        help="PR to comment on with stability results")
     parser.add_argument("browser",
                         action="store",
                         help="Browser to run against")
     return parser
 
 
 def main():
+    retcode = 0
     parser = get_parser()
     args = parser.parse_args()
 
+    gh_handler = setup_github_logging(args)
+
+    logger.info("Testing in %s" % args.browser)
+
     browser_cls = {"firefox": Firefox,
                    "chrome": Chrome}.get(args.browser)
     if browser_cls is None:
         logger.critical("Unrecognised browser %s" % args.browser)
-        sys.exit(2)
+        return 2
 
     # For now just pass the whole list of changed files to wptrunner and
     # assume that it will run everything that's actually a test
@@ -187,9 +247,21 @@ def main():
             logger.error("%s | %s | %s" % (test,
                                            subtest if subtest else "(parent)",
                                            err_string(results)))
-        sys.exit(1)
-    logger.info("All results were stable")
+        retcode = 1
+    else:
+        logger.info("All results were stable")
+    try:
+        if gh_handler:
+            gh_handler.send()
+    except Exception:
+        logger.error(traceback.format_exc())
+    return retcode
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        retcode = main()
+    except:
+        raise
+    else:
+        sys.exit(retcode)

ci_stability.sh

@@ -96,7 +96,7 @@ install_chromedriver() {
 
 test_stability() {
     cd $WPT_HOME
-    python check_stability.py --root $BUILD_HOME $PRODUCT
+    python check_stability.py --root $BUILD_HOME --comment-pr ${TRAVIS_PULL_REQUEST} --gh-token ${GH_TOKEN} $PRODUCT
 }
 
 main() {