Support running tests with SSL.

This adds support for launching a https server either using a
real CA-issued certificate, or a self-signed certificate generated
using OpenSSL. The latter in intended for most testing use cases.

.gitignore

@@ -6,4 +6,5 @@ scratch
 *#
 \#*
 config.json
-MANIFEST.json
+MANIFEST.json
+_certs

README.md

@@ -8,10 +8,12 @@ the Web Apps Security Working Group.
 Running the Tests
 =================
 
-The tests are designed to be run from your local computer. The test environment
-requires Python 2.7+ (but not Python 3.x).
+The tests are designed to be run from your local computer. The test
+environment requires Python 2.7+ (but not Python 3.x). You will also
+need a copy of OpenSSL. For users on Windows this is available from
+[the openssl website](https://www.openssl.org/related/binaries.html).
 
-To get the tests running, you need to set up the test domains in your 
+To get the tests running, you need to set up the test domains in your
 [`hosts` file](http://en.wikipedia.org/wiki/Hosts_%28file%29%23Location_in_the_file_system). The following entries are required:
 
 ```
@@ -52,6 +54,15 @@ to some port of your choice e.g.
 "http":[1234, "auto"]
 ```
 
+If you installed OpenSSL in such a way that running `openssl` at a
+command line doesn't work, you also need to adjust the path to the
+OpenSSL binary. This can be done by adding a section to `config.json`
+like:
+
+```
+"ssl": {"openssl": {"binary": "/path/to/openssl"}}
+```
+
 Test Runner
 ===========
 

config.default.json

@@ -1,8 +1,21 @@
 {"host": "web-platform.test",
  "external_host": null,
  "ports":{"http":[8000, "auto"],
-          "https":[],
+          "https":[8443],
           "ws":["auto"]},
  "check_subdomains": true,
  "log_level":"debug",
- "bind_hostname": true}
+ "bind_hostname": true,
+ "ssl": {"type": "openssl",
+         "openssl": {
+             "openssl_binary": "openssl",
+             "base_path": "_certs",
+             "force_regenerate": false
+         },
+         "pregenerated": {
+             "host_key_path": null,
+             "host_cert_path": null
+         },
+         "none": {}
+        }
+}

docs/configuration.md

@@ -6,6 +6,7 @@ the web-platform-tests repository.
 
  * [git](http://git-scm.com/)
  * [Python 2.7](http://python.org)
+ * [OpenSSL](https://www.openssl.org)
 
 ## Hosts configuration
 
@@ -66,3 +67,12 @@ to some port of your choice e.g.
 ```
 "http": [1234, "auto"]
 ```
+
+If you installed OpenSSL in such a way that running `openssl` at a
+command line doesn't work, you also need to adjust the path to the
+OpenSSL binary. This can be done by adding a section to `config.json`
+like:
+
+```
+"ssl": {"openssl": {"binary": "/path/to/openssl"}}
+```

serve.py

@@ -19,11 +19,16 @@
 sys.path.insert(1, os.path.join(repo_root, "tools", "wptserve"))
 from wptserve import server as wptserve, handlers
 from wptserve.router import any_method
+
 sys.path.insert(1, os.path.join(repo_root, "tools", "pywebsocket", "src"))
 from mod_pywebsocket import standalone as pywebsocket
 
+sys.path.insert(1, os.path.join(repo_root, "tools"))
+import sslutils
+
 routes = [("GET", "/tools/runner/*", handlers.file_handler),
           ("POST", "/tools/runner/update_manifest.py", handlers.python_script_handler),
+          (any_method, "/_certs/*", handlers.ErrorHandler(404)),
           (any_method, "/tools/*", handlers.ErrorHandler(404)),
           (any_method, "/serve.py", handlers.ErrorHandler(404)),
           (any_method, "*.py", handlers.python_script_handler),
@@ -46,6 +51,7 @@ def default_logger(level):
     logging.basicConfig(level=getattr(logging, level.upper()))
     return logger
 
+
 def open_socket(port):
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     if port != 0:
@@ -54,6 +60,7 @@ def open_socket(port):
     sock.listen(5)
     return sock
 
+
 def get_port():
     free_socket = open_socket(0)
     port = free_socket.getsockname()[1]
@@ -68,15 +75,18 @@ def __init__(self):
         self.daemon = None
         self.stop = Event()
 
-    def start(self, init_func, host, port, paths, bind_hostname, external_config):
+    def start(self, init_func, config, paths, port, bind_hostname, external_config, ssl_config):
         self.proc = Process(target=self.create_daemon,
-                            args=(init_func, host, port, paths, bind_hostname, external_config))
+                            args=(init_func, config, paths, port,
+                                  bind_hostname, external_config, ssl_config))
         self.proc.daemon = True
         self.proc.start()
 
-    def create_daemon(self, init_func, host, port, paths, bind_hostname, external_config):
+    def create_daemon(self, init_func, config, paths, port, bind_hostname, external_config,
+                      ssl_config):
         try:
-            self.daemon = init_func(host, port, paths, bind_hostname, external_config)
+            self.daemon = init_func(config, paths, port, bind_hostname, external_config,
+                                    ssl_config)
         except socket.error:
             print >> sys.stderr, "Socket error on port %s" % port
             raise
@@ -85,11 +95,15 @@ def create_daemon(self, init_func, host, port, paths, bind_hostname, external_co
             raise
 
         if self.daemon:
-            self.daemon.start(block=False)
             try:
-                self.stop.wait()
-            except KeyboardInterrupt:
-                pass
+                self.daemon.start(block=False)
+                try:
+                    self.stop.wait()
+                except KeyboardInterrupt:
+                    pass
+            except:
+                print >> sys.stderr, traceback.format_exc()
+                raise
 
     def wait(self):
         self.stop.set()
@@ -103,12 +117,13 @@ def kill(self):
     def is_alive(self):
         return self.proc.is_alive()
 
-def check_subdomains(host, paths, bind_hostname):
+
+def check_subdomains(host, paths, bind_hostname, ssl_config):
     port = get_port()
     subdomains = get_subdomains(host)
 
     wrapper = ServerProc()
-    wrapper.start(start_http_server, host, port, paths, bind_hostname, None)
+    wrapper.start(start_http_server, host, port, paths, bind_hostname, None, ssl_config)
 
     connected = False
     for i in range(10):
@@ -133,45 +148,63 @@ def check_subdomains(host, paths, bind_hostname):
 
     wrapper.wait()
 
+
 def get_subdomains(host):
     #This assumes that the tld is ascii-only or already in punycode
     return {subdomain: (subdomain.encode("idna"), host)
             for subdomain in subdomains}
 
-def start_servers(host, ports, paths, bind_hostname, external_config):
+
+def start_servers(host, ports, paths, bind_hostname, external_config, ssl_config):
     servers = defaultdict(list)
 
     for scheme, ports in ports.iteritems():
         assert len(ports) == {"http":2}.get(scheme, 1)
 
         for port  in ports:
+            if port is None:
+                continue
             init_func = {"http":start_http_server,
                          "https":start_https_server,
                          "ws":start_ws_server,
                          "wss":start_wss_server}[scheme]
 
             server_proc = ServerProc()
-            server_proc.start(init_func, host, port, paths, bind_hostname, external_config)
+            server_proc.start(init_func, host, port, paths, bind_hostname, external_config, ssl_config)
             servers[scheme].append((port, server_proc))
 
     return servers
 
-def start_http_server(host, port, paths, bind_hostname, external_config):
-    return wptserve.WebTestHttpd(host=host,
+
+def start_http_server(host, port, paths, bind_hostname, external_config, ssl_config):
+    return wptserve.WebTestHttpd(host,
                                  port=port,
                                  doc_root=paths["doc_root"],
                                  routes=routes,
                                  rewrites=rewrites,
                                  bind_hostname=bind_hostname,
                                  config=external_config,
                                  use_ssl=False,
+                                 key_file=None,
                                  certificate=None)
 
-def start_https_server(host, port, paths, bind_hostname):
-    return
+
+def start_https_server(host, port, paths, bind_hostname, external_config, ssl_config):
+    return wptserve.WebTestHttpd(host=host,
+                                 port=port,
+                                 doc_root=paths["doc_root"],
+                                 routes=routes,
+                                 rewrites=rewrites,
+                                 bind_hostname=bind_hostname,
+                                 config=external_config,
+                                 use_ssl=True,
+                                 key_file=ssl_config["key_path"],
+                                 certificate=ssl_config["cert_path"])
+
 
 class WebSocketDaemon(object):
-    def __init__(self, host, port, doc_root, handlers_root, log_level, bind_hostname):
+    def __init__(self, host, port, doc_root, handlers_root, log_level, bind_hostname,
+                 ssl_config):
         self.host = host
         cmd_args = ["-p", port,
                     "-d", doc_root,
@@ -215,31 +248,41 @@ def stop(self):
             self.started = False
         self.server = None
 
-def start_ws_server(host, port, paths, bind_hostname, external_config):
+
+def start_ws_server(host, port, paths, bind_hostname, external_config, ssl_config):
     return WebSocketDaemon(host,
                            str(port),
                            repo_root,
                            paths["ws_doc_root"],
                            "debug",
-                           bind_hostname)
+                           bind_hostname,
+                           ssl_config)
 
-def start_wss_server(host, port, path, bind_hostname, external_config):
+def start_wss_server(host, port, paths, bind_hostname, external_config, ssl_config):
     return
 
-def get_ports(config):
+
+def get_ports(config, ssl_enabled):
     rv = defaultdict(list)
     for scheme, ports in config["ports"].iteritems():
         for i, port in enumerate(ports):
+            if scheme in ["http", "https"] and not ssl_enabled:
+                port = None
             if port == "auto":
                 port = get_port()
             else:
                 port = port
             rv[scheme].append(port)
     return rv
 
+
+
 def normalise_config(config, ports):
     host = config["external_host"] if config["external_host"] else config["host"]
     domains = get_subdomains(host)
+    ports_ = {}
+    for scheme, ports_used in ports.iteritems():
+        ports_[scheme] = ports_used
 
     for key, value in domains.iteritems():
         domains[key] = ".".join(value)
@@ -258,17 +301,34 @@ def start(config):
     host = config["host"]
     domains = get_subdomains(host)
     ports = get_ports(config)
+    return {"host":config["host"],
+            "domains":domains_,
+            "ports": ports_}
+
+
+def get_ssl_config(external_domains, ssl_environment):
+    key_path, cert_path = ssl_environment.host_cert_path(external_domains)
+    return {"key_path": key_path,
+            "cert_path": cert_path}
+
+
+def start(config, ssl_environment):
+    host = config["host"]
+    domains = get_subdomains(host)
+    ports = get_ports(config, ssl_environment)
     bind_hostname = config["bind_hostname"]
 
     paths = {"doc_root": config["doc_root"],
              "ws_doc_root": config["ws_doc_root"]}
 
-    if config["check_subdomains"]:
-        check_subdomains(host, paths, bind_hostname)
-
     external_config = normalise_config(config, ports)
 
-    servers = start_servers(host, ports, paths, bind_hostname, external_config)
+    ssl_config = get_ssl_config(external_config["domains"].values(), ssl_environment)
+
+    if config["check_subdomains"]:
+        check_subdomains(host, paths, bind_hostname, ssl_config)
+
+    servers = start_servers(host, ports, paths, bind_hostname, external_config, ssl_config)
 
     return external_config, servers
 
@@ -278,9 +338,11 @@ def iter_procs(servers):
         for port, server in servers:
             yield server.proc
 
+
 def value_set(config, key):
     return key in config and config[key] is not None
 
+
 def set_computed_defaults(config):
     if not value_set(config, "ws_doc_root"):
         if value_set(config, "doc_root"):
@@ -305,6 +367,16 @@ def merge_json(base_obj, override_obj):
                 rv[key] = override_obj[key]
     return rv
 
+
+def get_ssl_environment(config):
+    implementation_type = config["ssl"]["type"]
+    cls = {"none": sslutils.NoSSLEnvironment,
+           "openssl": sslutils.OpenSSLEnvironment,
+           "pregenerated": sslutils.PregeneratedSSLEnvironment}[implementation_type]
+    kwargs = config["ssl"][implementation_type].copy()
+    return cls(logger, **kwargs)
+
+
 def load_config(default_path, override_path=None):
     if os.path.exists(default_path):
         with open(default_path) as f:
@@ -322,6 +394,7 @@ def load_config(default_path, override_path=None):
     set_computed_defaults(rv)
     return rv
 
+
 def main():
     global logger
 
@@ -330,14 +403,18 @@ def main():
 
     logger = default_logger(config["log_level"])
 
-    config_, servers = start(config)
+    ssl_env = get_ssl_environment(config)
+
+    with get_ssl_environment(config) as ssl_env:
+        config_, servers = start(config, ssl_env)
+
+        try:
+            while any(item.is_alive() for item in iter_procs(servers)):
+                for item in iter_procs(servers):
+                    item.join(1)
+        except KeyboardInterrupt:
+            logger.info("Shutting down")
 
-    try:
-        while any(item.is_alive() for item in iter_procs(servers)):
-            for item in iter_procs(servers):
-                item.join(1)
-    except KeyboardInterrupt:
-        logger.info("Shutting down")
 
 if __name__ == "__main__":
     main()