Allow extending traefik dynamic config

docs/topic/escape-hatch.rst

@@ -8,11 +8,19 @@ Custom configuration snippets
 The two main TLJH components are **JupyterHub** and **Traefik**.
 
 * JupyterHub takes its configuration from the ``jupyterhub_config.py`` file.
-* Traefik takes its configuration from the ``traefik.toml`` file.
+* Traefik loads its:
+	* `static configuration <https://docs.traefik.io/v1.7/basics/#static-traefik-configuration>`_
+	  from the ``traefik.toml`` file.
+	* `dynamic configuration <https://docs.traefik.io/v1.7/basics/#dynamic-traefik-configuration>`_
+	  from the ``rules`` directory.
 
-These files are created by TLJH during installation and can be edited by the
-user only through ``tljh-config``. Any direct modification to these files
-is unsupported, and will cause hard to debug issues.
+The ``jupyterhub_config.py`` and ``traefik.toml`` files are created by TLJH during installation
+and can be edited by the user only through ``tljh-config``. The ``rules`` directory is also created
+during install along with a ``rules/rules.toml`` file, to be used by JupyterHub to store the routing
+table from users to their notebooks.
+
+.. note::
+	Any direct modification to these files is unsupported, and will cause hard to debug issues.
 
 But because sometimes TLJH needs to be customized in ways that are not officially
 supported, an escape hatch has been introduced to allow easily extending the
@@ -65,3 +73,22 @@ proxy for the new configuration to take effect:
 	sudo tljh-config reload proxy
 
 .. warning:: This instructions might change when TLJH will switch to Traefik > 2.0
+
+Extending ``rules.toml``
+========================
+
+``Traefik`` is configured to load its routing table from the ``/opt/tljh/state/rules``
+directory. The existing ``rules.toml`` file inside this directory is used by
+``jupyterhub-traefik-proxy`` to add the JupyterHub routes from users to their notebook servers
+and shouldn't be modified.
+
+However, the routing table can be extended  outside JupyterHub's scope using the ``rules``
+directory, by adding other dynamic configuration files with the desired routing rules.
+
+.. note::
+	* Any files in ``/opt/tljh/state/rules`` that end in ``.toml`` will be hot reload by Traefik.
+	  This means that there is no need to reload the proxy service for the rules to take effect.
+
+Checkout Traefik' docs about `dynamic configuration <https://docs.traefik.io/v1.7/basics/#dynamic-traefik-configuration>`_
+and how to provide dynamic configuration through
+`multiple separated files <https://docs.traefik.io/v1.7/configuration/backends/file/#multiple-separated-files>`_.

integration-tests/test_proxy.py

@@ -9,7 +9,37 @@
 from tornado.httpclient import HTTPClient, HTTPRequest, HTTPClientError
 import pytest
 
-from tljh.config import reload_component, set_config_value, CONFIG_FILE, CONFIG_DIR
+from tljh.config import (
+    reload_component,
+    set_config_value,
+    unset_config_value,
+    CONFIG_FILE,
+    CONFIG_DIR,
+    STATE_DIR,
+)
+
+
+def send_request(url, max_sleep, validate_cert=True, username=None, password=None):
+    resp = None
+    for i in range(max_sleep):
+        time.sleep(i)
+        try:
+            req = HTTPRequest(
+                url,
+                method="GET",
+                auth_username=username,
+                auth_password=password,
+                validate_cert=validate_cert,
+                follow_redirects=True,
+                max_redirects=15,
+            )
+            resp = HTTPClient().fetch(req)
+            break
+        except Exception as e:
+            print(e)
+            pass
+
+    return resp
 
 
 def test_manual_https(preserve_config):
@@ -55,32 +85,49 @@ def test_manual_https(preserve_config):
     # verify that our certificate was loaded by traefik
     assert server_cert == file_cert
 
-    for i in range(10):
-        time.sleep(i)
-        # verify that we can still connect to the hub
-        try:
-            req = HTTPRequest(
-                "https://127.0.0.1/hub/api", method="GET", validate_cert=False
-            )
-            resp = HTTPClient().fetch(req)
-            break
-        except Exception as e:
-            pass
+    # verify that we can still connect to the hub
+    resp = send_request(
+        url="https://127.0.0.1/hub/api", max_sleep=10, validate_cert=False
+    )
     assert resp.code == 200
 
+
     # cleanup
     shutil.rmtree(ssl_dir)
+    set_config_value(CONFIG_FILE, "https.enabled", False)
+
+    reload_component("proxy")
 
 
 def test_extra_traefik_config():
-    extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d")
-    os.makedirs(extra_config_dir, exist_ok=True)
+    extra_static_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d")
+    os.makedirs(extra_static_config_dir, exist_ok=True)
+
+    dynamic_config_dir = os.path.join(STATE_DIR, "rules")
+    os.makedirs(dynamic_config_dir, exist_ok=True)
 
-    extra_config = {
+
+    extra_static_config = {
         "entryPoints": {"no_auth_api": {"address": "127.0.0.1:9999"}},
         "api": {"dashboard": True, "entrypoint": "no_auth_api"},
     }
 
+    extra_dynamic_config = {
+        "frontends": {
+            "test": {
+                "backend": "test",
+                "routes": {
+                    "rule1": {"rule": "PathPrefixStrip: /the/hub/runs/here/too"}
+                },
+            }
+        },
+        "backends": {
+            # redirect to hub
+            "test": {"servers": {"server1": {"url": "http://127.0.0.1:15001"}}}
+        },
+    }
+
+
     success = False
     for i in range(5):
         time.sleep(i)
@@ -96,23 +143,31 @@ def test_extra_traefik_config():
 
     assert success == True
 
-    # Load the extra config
-    with open(os.path.join(extra_config_dir, "extra.toml"), "w+") as extra_config_file:
-        toml.dump(extra_config, extra_config_file)
+    # write the extra static config
+    with open(
+        os.path.join(extra_static_config_dir, "extra.toml"), "w+"
+    ) as extra_config_file:
+        toml.dump(extra_static_config, extra_config_file)
+
+    # write the extra dynamic config
+    with open(
+        os.path.join(dynamic_config_dir, "extra_rules.toml"), "w+"
+    ) as extra_config_file:
+        toml.dump(extra_dynamic_config, extra_config_file)
+
+    # load the extra config
     reload_component("proxy")
 
-    for i in range(5):
-        time.sleep(i)
-        try:
-            # The new dashboard entrypoint shouldn't require authentication anymore
-            req = HTTPRequest("http://127.0.0.1:9999/dashboard/", method="GET")
-            resp = HTTPClient().fetch(req)
-            break
-        except ConnectionRefusedError:
-            pass
-    # If the request didn't get through after 5 tries, this should fail
+    # the new dashboard entrypoint shouldn't require authentication anymore
+    resp = send_request(url="http://127.0.0.1:9999/dashboard/", max_sleep=5)
     assert resp.code == 200
 
+    # test extra dynamic config
+    resp = send_request(url="http://127.0.0.1/the/hub/runs/here/too", max_sleep=5)
+    assert resp.code == 200
+    assert resp.effective_url == "http://127.0.0.1/hub/login"
+
     # cleanup
-    os.remove(os.path.join(extra_config_dir, "extra.toml"))
-    reload_component("proxy")
+    os.remove(os.path.join(extra_static_config_dir, "extra.toml"))
+    os.remove(os.path.join(dynamic_config_dir, "extra_rules.toml"))
+    open(os.path.join(STATE_DIR, "traefik.toml"), "w").close()

tljh/jupyterhub_config.py

@@ -53,7 +53,7 @@ def start(self):
 
 c.TraefikTomlProxy.should_start = False
 
-dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules.toml')
+dynamic_conf_file_path = os.path.join(INSTALL_PREFIX, 'state', 'rules', 'rules.toml')
 c.TraefikTomlProxy.toml_dynamic_config_file = dynamic_conf_file_path
 c.JupyterHub.proxy_class = TraefikTomlProxy
 

tljh/systemd-units/traefik.service

@@ -14,7 +14,7 @@ PrivateTmp=yes
 PrivateDevices=yes
 ProtectKernelTunables=yes
 ProtectKernelModules=yes
-ReadWritePaths={install_prefix}/state/rules.toml
+ReadWritePaths={install_prefix}/state/rules
 ReadWritePaths={install_prefix}/state/acme.json
 WorkingDirectory={install_prefix}/state
 ExecStart={install_prefix}/hub/bin/traefik \

tljh/traefik.py

@@ -92,6 +92,7 @@ def ensure_traefik_config(state_dir):
     """Render the traefik.toml config file"""
     traefik_std_config_file = os.path.join(state_dir, "traefik.toml")
     traefik_extra_config_dir = os.path.join(CONFIG_DIR, "traefik_config.d")
+    traefik_dynamic_config_dir = os.path.join(state_dir, "rules")
 
     config = load_config()
     config['traefik_api']['basic_auth'] = compute_basic_auth(
@@ -116,9 +117,12 @@ def ensure_traefik_config(state_dir):
         ):
             raise ValueError("Both email and domains must be set for letsencrypt")
 
-    # Ensure extra config dir exists and is private
+    # Ensure traefik extra static config dir exists and is private
     os.makedirs(traefik_extra_config_dir, mode=0o700, exist_ok=True)
 
+    # Ensure traefik dynamic config dir exists and is private
+    os.makedirs(traefik_dynamic_config_dir, mode=0o700, exist_ok=True)
+
     try:
         # Load standard config file merge it with the extra config files into a dict
         extra_config = load_extra_config(traefik_extra_config_dir)
@@ -131,7 +135,7 @@ def ensure_traefik_config(state_dir):
         os.fchmod(f.fileno(), 0o600)
         toml.dump(new_toml, f)
 
-    with open(os.path.join(state_dir, "rules.toml"), "w") as f:
+    with open(os.path.join(traefik_dynamic_config_dir, "rules.toml"), "w") as f:
         os.fchmod(f.fileno(), 0o600)
 
     # ensure acme.json exists and is private

tljh/traefik.toml.tpl

@@ -70,5 +70,5 @@ entryPoint = "https"
 {% endif %}
 
 [file]
-filename = "rules.toml"
+directory = "rules"
 watch = true