Add "quic" property to manifest and add "--include-quic" switch to wptrunner

docs/writing-tests/server-features.md

@@ -18,10 +18,12 @@ To make writing such tests possible, we are using a number of
 server-side components designed to make it easy to manipulate the
 precise details of the response:
 
-* *wptserve*, a custom Python HTTP server.
+* *wptserve*, a custom Python HTTP server
 
 * *pywebsocket*, an existing websockets server
 
+* *tools/quic*, a custom Python 3 QUIC server using `aioquic`
+
 wptserve is a Python-based web server. By default it serves static
 files in the test suite. For more sophisticated requirements, several
 mechanisms are available to take control of the response. These are
@@ -116,7 +118,8 @@ The server also provides the ability to write [Python
 data and can manipulate the content and timing of the response. Responses are
 also influenced by [the `pipe` query string parameter](server-pipes).
 
-### Writing tests for HTTP/2.0
+
+### Tests Requiring HTTP/2.0
 
 The server now has a prototype HTTP/2.0 server which gives you access to
 some of the HTTP/2.0 specific functionality. Currently, the server is off
@@ -126,4 +129,27 @@ API are documented in [Writing H2 Tests](h2tests).
 
 > <b>Important:</b> The HTTP/2.0 server requires you to have Python 2.7.10+
 and OpenSSL 1.0.2+. This is because HTTP/2.0 is negotiated using the
-[TLS ALPN](https://tools.ietf.org/html/rfc7301) extension, which is only supported in [OpenSSL 1.0.2](https://www.openssl.org/news/openssl-1.0.2-notes.html) and up.
+[TLS ALPN](https://tools.ietf.org/html/rfc7301) extension, which is only
+supported in
+[OpenSSL 1.0.2](https://www.openssl.org/news/openssl-1.0.2-notes.html) and up.
+
+
+### Tests Requiring QUIC
+
+We do not support loading a test over QUIC yet, but a test can establish a QUIC
+connection to the test server (e.g. for WebTransport, similar to WebSocket).
+Since the QUIC server is not yet enabled by default, tests must explicitly
+declare that they need access to the QUIC server:
+
+* For HTML tests (including testharness.js and reference tests), add the
+  following element:
+```html
+<meta name="quic" content="true">
+```
+* For JavaScript tests (auto-generated tests), add the following comment:
+```js
+// META: quic=true
+```
+
+The QUIC server is not yet enabled by default, so QUIC tests will be skipped
+unless `--enable-quic` is specified to `./wpt run`.

tools/manifest/item.py

@@ -190,6 +190,11 @@ def jsshell(self):
         # type: () -> Optional[Text]
         return self._extras.get("jsshell")
 
+    @property
+    def quic(self):
+        # type: () -> Optional[bool]
+        return self._extras.get("quic")
+
     @property
     def script_metadata(self):
         # type: () -> Optional[Text]
@@ -204,6 +209,8 @@ def to_json(self):
             rv[-1]["testdriver"] = self.testdriver
         if self.jsshell:
             rv[-1]["jsshell"] = True
+        if self.quic is not None:
+            rv[-1]["quic"] = self.quic
         if self.script_metadata:
             rv[-1]["script_metadata"] = [(k.decode('utf8'), v.decode('utf8')) for (k,v) in self.script_metadata]
         return rv

tools/manifest/sourcefile.py

@@ -674,6 +674,36 @@ def has_testdriver(self):
             return None
         return bool(self.testdriver_nodes)
 
+    @cached_property
+    def quic_nodes(self):
+        # type: () -> List[ElementTree.Element]
+        """List of ElementTree Elements corresponding to nodes in a test that
+        specify whether it needs QUIC server."""
+        assert self.root is not None
+        return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='quic']")
+
+    @cached_property
+    def quic(self):
+        # type: () -> Optional[bool]
+        """Boolean indicating whether a test requires QUIC server
+
+        Determined by <meta> elements (`quic_nodes()`) and "// META" comments
+        (`script_metadata()`).
+        """
+        if self.script_metadata:
+            if any(m == (b"quic", b"true") for m in self.script_metadata):
+                return True
+
+        if self.root is None:
+            return None
+
+        if self.quic_nodes:
+            quic_str = self.quic_nodes[0].attrib.get("content", "false")  # type: Text
+            if quic_str.lower() == "true":
+                return True
+
+        return None
+
     @cached_property
     def reftest_nodes(self):
         # type: () -> List[ElementTree.Element]
@@ -850,6 +880,7 @@ def manifest_items(self):
                     global_variant_url(self.rel_url, suffix) + variant,
                     timeout=self.timeout,
                     jsshell=jsshell,
+                    quic=self.quic,
                     script_metadata=self.script_metadata
                 )
                 for (suffix, jsshell) in sorted(global_suffixes(globals))
@@ -866,6 +897,7 @@ def manifest_items(self):
                     self.url_base,
                     test_url + variant,
                     timeout=self.timeout,
+                    quic=self.quic,
                     script_metadata=self.script_metadata
                 )
                 for variant in self.test_variants
@@ -881,6 +913,7 @@ def manifest_items(self):
                     self.url_base,
                     test_url + variant,
                     timeout=self.timeout,
+                    quic=self.quic,
                     script_metadata=self.script_metadata
                 )
                 for variant in self.test_variants
@@ -917,6 +950,7 @@ def manifest_items(self):
                     self.url_base,
                     url,
                     timeout=self.timeout,
+                    quic=self.quic,
                     testdriver=testdriver,
                     script_metadata=self.script_metadata
                 ))
@@ -930,6 +964,7 @@ def manifest_items(self):
                     self.rel_url,
                     references=self.references,
                     timeout=self.timeout,
+                    quic=self.quic,
                     viewport_size=self.viewport_size,
                     dpi=self.dpi,
                     fuzzy=self.fuzzy

tools/manifest/tests/test_sourcefile.py

@@ -6,6 +6,7 @@
 from ...lint.lint import check_global_metadata
 from ..sourcefile import SourceFile, read_script_metadata, js_meta_re, python_meta_re
 
+
 def create(filename, contents=b""):
     assert isinstance(contents, bytes)
     return SourceFile("/", filename, "/", contents=contents)
@@ -144,6 +145,8 @@ def test_worker():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
+
 
 def test_window():
     s = create("html/test.window.js")
@@ -168,6 +171,7 @@ def test_window():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_worker_long_timeout():
@@ -233,6 +237,7 @@ def test_worker_with_variants():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_window_with_variants():
@@ -263,6 +268,7 @@ def test_window_with_variants():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_python_long_timeout():
@@ -307,6 +313,7 @@ def test_multi_global():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_multi_global_long_timeout():
@@ -366,6 +373,7 @@ def test_multi_global_with_custom_globals(input, expected):
         assert item.url == url
         assert item.jsshell is False
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_multi_global_with_jsshell_globals():
@@ -396,6 +404,7 @@ def test_multi_global_with_jsshell_globals():
         assert item.url == url
         assert item.jsshell == jsshell
         assert item.timeout is None
+        assert item.quic is None
 
 
 def test_multi_global_with_variants():
@@ -434,6 +443,7 @@ def test_multi_global_with_variants():
     for item, url in zip(items, expected_urls):
         assert item.url == url
         assert item.timeout is None
+        assert item.quic is None
 
 
 @pytest.mark.parametrize("input,expected", [
@@ -642,6 +652,35 @@ def test_relative_testdriver(ext):
     assert not s.has_testdriver
 
 
+@pytest.mark.parametrize("ext", ["htm", "html"])
+def test_quic_html(ext):
+    filename = "html/test." + ext
+
+    content = b'<meta name="quic" content="true">'
+    s = create(filename, content)
+    assert s.quic
+
+    content = b'<meta name="quic" content="false">'
+    s = create(filename, content)
+    assert s.quic is None
+
+
+def test_quic_js():
+    filename = "html/test.any.js"
+
+    content = b"// META: quic=true"
+    s = create(filename, content)
+    _, items = s.manifest_items()
+    for item in items:
+        assert item.quic
+
+    content = b"// META: quic=false"
+    s = create(filename, content)
+    _, items = s.manifest_items()
+    for item in items:
+        assert item.quic is None
+
+
 @pytest.mark.parametrize("ext", ["htm", "html"])
 def test_reftest(ext):
     content = b"<link rel=match href=ref.html>"

tools/wptrunner/wptrunner/environment.py

@@ -53,7 +53,9 @@ class TestEnvironmentError(Exception):
 class TestEnvironment(object):
     """Context manager that owns the test environment i.e. the http and
     websockets servers"""
-    def __init__(self, test_paths, testharness_timeout_multipler, pause_after_test, debug_info, options, ssl_config, env_extras):
+    def __init__(self, test_paths, testharness_timeout_multipler,
+                 pause_after_test, debug_info, options, ssl_config, env_extras,
+                 enable_quic=False):
         self.test_paths = test_paths
         self.server = None
         self.config_ctx = None
@@ -69,6 +71,7 @@ def __init__(self, test_paths, testharness_timeout_multipler, pause_after_test,
         self.env_extras = env_extras
         self.env_extras_cms = None
         self.ssl_config = ssl_config
+        self.enable_quic = enable_quic
 
     def __enter__(self):
         self.config_ctx = self.build_config()
@@ -130,6 +133,8 @@ def build_config(self):
             "wss": [8889],
             "h2": [9000],
         }
+        if self.enable_quic:
+            config.ports["quic"] = [10000]
 
         if os.path.exists(override_path):
             with open(override_path) as f:

tools/wptrunner/wptrunner/testloader.py

@@ -168,6 +168,7 @@ def __init__(self,
                  total_chunks=1,
                  chunk_number=1,
                  include_https=True,
+                 include_quic=False,
                  skip_timeout=False,
                  skip_implementation_status=None,
                  chunker_kwargs=None):
@@ -181,6 +182,7 @@ def __init__(self,
         self.tests = None
         self.disabled_tests = None
         self.include_https = include_https
+        self.include_quic = include_quic
         self.skip_timeout = skip_timeout
         self.skip_implementation_status = skip_implementation_status
 
@@ -267,6 +269,8 @@ def _load_tests(self):
             enabled = not test.disabled()
             if not self.include_https and test.environment["protocol"] == "https":
                 enabled = False
+            if not self.include_quic and test.environment["quic"]:
+                enabled = False
             if self.skip_timeout and test.expected() == "TIMEOUT":
                 enabled = False
             if self.skip_implementation_status and test.implementation_status() in self.skip_implementation_status:

tools/wptrunner/wptrunner/wptcommandline.py

@@ -141,6 +141,10 @@ def create_parser(product_choices=None):
                                       action="append",
                                       choices=["not-implementing", "backlog", "implementing"],
                                       help="Skip tests that have the given implementation status")
+    # TODO: Remove this when QUIC is enabled by default.
+    test_selection_group.add_argument("--enable-quic", action="store_true", default=False,
+                                      help="Enable tests that require QUIC server (default: false)")
+
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. "
                                            "Labels starting dir: are equivalent to top-level directories.")

tools/wptrunner/wptrunner/wptrunner.py

@@ -78,6 +78,7 @@ def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kw
                                         total_chunks=kwargs["total_chunks"],
                                         chunk_number=kwargs["this_chunk"],
                                         include_https=ssl_enabled,
+                                        include_quic=kwargs["enable_quic"],
                                         skip_timeout=kwargs["skip_timeout"],
                                         skip_implementation_status=kwargs["skip_implementation_status"],
                                         chunker_kwargs=chunker_kwargs)
@@ -211,7 +212,8 @@ def run_tests(config, test_paths, product, **kwargs):
                                  kwargs["debug_info"],
                                  product.env_options,
                                  ssl_config,
-                                 env_extras) as test_environment:
+                                 env_extras,
+                                 kwargs["enable_quic"]) as test_environment:
             recording.set(["startup", "ensure_environment"])
             try:
                 test_environment.ensure_started()