[wptrunner] Add a switch to include QUIC tests

--include-quic, set to false by default.

This flag also controls whether to assign a port to QUIC in the server
config, which will then be passed to `wpt serve` to determine whether to
start the QUIC server (to be implemented).

tools/wptrunner/wptrunner/environment.py

@@ -53,7 +53,7 @@ class TestEnvironmentError(Exception):
 class TestEnvironment(object):
     """Context manager that owns the test environment i.e. the http and
     websockets servers"""
-    def __init__(self, test_paths, testharness_timeout_multipler, pause_after_test, debug_info, options, ssl_config, env_extras):
+    def __init__(self, test_paths, testharness_timeout_multipler, pause_after_test, debug_info, options, ssl_config, env_extras, enable_quic=False):
         self.test_paths = test_paths
         self.server = None
         self.config_ctx = None
@@ -69,6 +69,7 @@ def __init__(self, test_paths, testharness_timeout_multipler, pause_after_test,
         self.env_extras = env_extras
         self.env_extras_cms = None
         self.ssl_config = ssl_config
+        self.enable_quic = enable_quic
 
     def __enter__(self):
         self.config_ctx = self.build_config()
@@ -130,6 +131,8 @@ def build_config(self):
             "wss": [8889],
             "h2": [9000],
         }
+        if self.enable_quic:
+            config.ports["quic"]: [10000]
 
         if os.path.exists(override_path):
             with open(override_path) as f:

tools/wptrunner/wptrunner/testloader.py

@@ -168,6 +168,7 @@ def __init__(self,
                  total_chunks=1,
                  chunk_number=1,
                  include_https=True,
+                 include_quic=False,
                  skip_timeout=False,
                  skip_implementation_status=None,
                  chunker_kwargs=None):
@@ -181,6 +182,7 @@ def __init__(self,
         self.tests = None
         self.disabled_tests = None
         self.include_https = include_https
+        self.include_quic = include_quic
         self.skip_timeout = skip_timeout
         self.skip_implementation_status = skip_implementation_status
 
@@ -267,6 +269,8 @@ def _load_tests(self):
             enabled = not test.disabled()
             if not self.include_https and test.environment["protocol"] == "https":
                 enabled = False
+            if not self.include_quic and test.environment["quic"]:
+                enabled = False
             if self.skip_timeout and test.expected() == "TIMEOUT":
                 enabled = False
             if self.skip_implementation_status and test.implementation_status() in self.skip_implementation_status:

tools/wptrunner/wptrunner/wptcommandline.py

@@ -141,6 +141,10 @@ def create_parser(product_choices=None):
                                       action="append",
                                       choices=["not-implementing", "backlog", "implementing"],
                                       help="Skip tests that have the given implementation status")
+    # TODO: Remove this when QUIC is enabled by default.
+    test_selection_group.add_argument("--include-quic", action="store_true", default=False,
+                                      help="Include tests that require QUIC server (default: false)")
+
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. "
                                            "Labels starting dir: are equivalent to top-level directories.")

tools/wptrunner/wptrunner/wptrunner.py

@@ -78,6 +78,7 @@ def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kw
                                         total_chunks=kwargs["total_chunks"],
                                         chunk_number=kwargs["this_chunk"],
                                         include_https=ssl_enabled,
+                                        include_quic=kwargs["include_quic"],
                                         skip_timeout=kwargs["skip_timeout"],
                                         skip_implementation_status=kwargs["skip_implementation_status"],
                                         chunker_kwargs=chunker_kwargs)
@@ -211,7 +212,8 @@ def run_tests(config, test_paths, product, **kwargs):
                                  kwargs["debug_info"],
                                  product.env_options,
                                  ssl_config,
-                                 env_extras) as test_environment:
+                                 env_extras,
+                                 kwargs["include_quic"]) as test_environment:
             recording.set(["startup", "ensure_environment"])
             try:
                 test_environment.ensure_started()

tools/wptrunner/wptrunner/wpttest.py

@@ -124,7 +124,6 @@ def __init__(self, metadata_root, product, debug,
         self["headless"] = extras.get("headless", False)
         self["webrender"] = enable_webrender
 
-
     def _update_mozinfo(self, metadata_root):
         """Add extra build information from a mozinfo.json file in a parent
         directory"""
@@ -159,14 +158,14 @@ class Test(object):
     long_timeout = 60  # seconds
 
     def __init__(self, tests_root, url, inherit_metadata, test_metadata,
-                 timeout=None, path=None, protocol="http"):
+                 timeout=None, path=None, protocol="http", quic=False):
         self.tests_root = tests_root
         self.url = url
         self._inherit_metadata = inherit_metadata
         self._test_metadata = test_metadata
         self.timeout = timeout if timeout is not None else self.default_timeout
         self.path = path
-        self.environment = {"protocol": protocol, "prefs": self.prefs}
+        self.environment = {"protocol": protocol, "prefs": self.prefs, "quic": quic}
 
     def __eq__(self, other):
         if not isinstance(other, Test):
@@ -396,9 +395,9 @@ class TestharnessTest(Test):
 
     def __init__(self, tests_root, url, inherit_metadata, test_metadata,
                  timeout=None, path=None, protocol="http", testdriver=False,
-                 jsshell=False, scripts=None):
+                 jsshell=False, scripts=None, quic=False):
         Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
-                      path, protocol)
+                      path, protocol, quic)
 
         self.testdriver = testdriver
         self.jsshell = jsshell
@@ -409,6 +408,7 @@ def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_meta
         timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout
         testdriver = manifest_item.testdriver if hasattr(manifest_item, "testdriver") else False
         jsshell = manifest_item.jsshell if hasattr(manifest_item, "jsshell") else False
+        quic = manifest_item.quic if hasattr(manifest_item, "quic") else False
         script_metadata = manifest_item.script_metadata or []
         scripts = [v for (k, v) in script_metadata if k == b"script"]
         return cls(manifest_file.tests_root,
@@ -420,8 +420,8 @@ def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_meta
                    protocol=server_protocol(manifest_item),
                    testdriver=testdriver,
                    jsshell=jsshell,
-                   scripts=scripts
-                   )
+                   scripts=scripts,
+                   quic=quic)
 
     @property
     def id(self):
@@ -452,9 +452,9 @@ class ReftestTest(Test):
     test_type = "reftest"
 
     def __init__(self, tests_root, url, inherit_metadata, test_metadata, references,
-                 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None, protocol="http"):
+                 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None, protocol="http", quic=False):
         Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
-                      path, protocol)
+                      path, protocol, quic)
 
         for _, ref_type in references:
             if ref_type not in ("==", "!="):
@@ -473,6 +473,7 @@ def from_manifest(cls,
                       test_metadata):
 
         timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout
+        quic = manifest_test.quic if hasattr(manifest_test, "quic") else False
 
         url = manifest_test.url
 
@@ -486,7 +487,8 @@ def from_manifest(cls,
                    viewport_size=manifest_test.viewport_size,
                    dpi=manifest_test.dpi,
                    protocol=server_protocol(manifest_test),
-                   fuzzy=manifest_test.fuzzy)
+                   fuzzy=manifest_test.fuzzy,
+                   quic=quic)
 
         refs_by_type = defaultdict(list)