Merge pull request #266 from GeorgianaElena/replace_chp_traefik

Replace chp with traefik-proxy

.circleci/integration-test.py

@@ -108,7 +108,7 @@ def show_logs(container_name):
     )
     run_container_command(
         container_name,
-        'systemctl --no-pager status jupyterhub configurable-http-proxy'
+        'systemctl --no-pager status jupyterhub traefik'
     )
 
 def main():

dev-requirements.txt

@@ -1,4 +1,4 @@
 pytest
 pytest-cov
 codecov
-pytoml
+pytoml

integration-tests/test_hub.py

@@ -34,10 +34,6 @@ async def test_user_code_execute():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
-
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -62,9 +58,6 @@ async def test_user_admin_add():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -94,9 +87,6 @@ async def test_user_admin_remove():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
             await u.login()
             await u.ensure_server()
@@ -107,16 +97,14 @@ async def test_user_admin_remove():
             # Assert that the user has admin rights
             assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem
 
-
             assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'remove-item', 'users.admin', username)).wait()
             assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
-            await asyncio.sleep(1)
 
             await u.stop_server()
             await u.ensure_server()
 
             # Assert that the user does *not* have admin rights
-            assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem
+            assert f'jupyter-{username}' not in grp.getgrnam('jupyterhub-admins').gr_mem
 
 
 @pytest.mark.asyncio
@@ -132,9 +120,6 @@ async def test_long_username():
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
-    # FIXME: wait for reload to finish & hub to come up
-    # Should be part of tljh-config reload
-    await asyncio.sleep(1)
     try:
         async with User(username, hub_url, partial(login_dummy, password='')) as u:
                 await u.login()

integration-tests/test_proxy.py

@@ -53,8 +53,13 @@ def test_manual_https(preserve_config):
     # verify that our certificate was loaded by traefik
     assert server_cert == file_cert
 
-    # verify that we can still connect to the hub
-    r = requests.get("https://127.0.0.1/hub/api", verify=False)
+    for i in range(5):
+        time.sleep(i)
+        # verify that we can still connect to the hub
+        r = requests.get("https://127.0.0.1/hub/api", verify=False)
+        if r.status_code == 200:
+            break;
+
     r.raise_for_status()
 
     # cleanup

setup.py

@@ -13,7 +13,9 @@
     install_requires=[
         'ruamel.yaml==0.15.*',
         'jinja2',
-        'pluggy>0.7<1.0'
+        'pluggy>0.7<1.0',
+        'passlib',
+        'jupyterhub-traefik-proxy==0.1.*'
     ],
     entry_points={
         'console_scripts': [

tests/test_config.py

@@ -75,6 +75,7 @@ def test_add_to_config_zero_level():
         'a': ['b']
     }
 
+
 def test_add_to_config_multiple():
     conf = {}
 
@@ -116,16 +117,21 @@ def test_remove_from_config_error():
 
 
 def test_reload_hub():
-    with mock.patch('tljh.systemd.restart_service') as restart_service:
+    with mock.patch('tljh.systemd.restart_service') as restart_service, mock.patch(
+        'tljh.systemd.check_service_active'
+    ) as check_active, mock.patch('tljh.config.check_hub_ready') as check_ready:
         config.reload_component('hub')
     assert restart_service.called_with('jupyterhub')
+    assert check_active.called_with('jupyterhub')
 
 
 def test_reload_proxy(tljh_dir):
-    with mock.patch('tljh.systemd.restart_service') as restart_service:
+    with mock.patch("tljh.systemd.restart_service") as restart_service, mock.patch(
+        "tljh.systemd.check_service_active"
+    ) as check_active:
         config.reload_component('proxy')
-    assert restart_service.called_with('configurable-http-proxy')
     assert restart_service.called_with('traefik')
+    assert check_active.called_with('traefik')
     assert os.path.exists(os.path.join(config.STATE_DIR, 'traefik.toml'))
 
 
@@ -140,8 +146,8 @@ def test_cli_no_command(capsys):
     "arg, value",
     [
         ("true", True),
-        ("FALSE", False),
-    ],
+        ("FALSE", False)
+    ]
 )
 def test_cli_set_bool(tljh_dir, arg, value):
     config.main(["set", "https.enabled", arg])

tests/test_configurer.py

@@ -1,7 +1,9 @@
 """
-Test
+Test configurer
 """
 
+import os
+
 from tljh import configurer
 
 
@@ -161,6 +163,43 @@ def test_auth_github():
     assert c.GitHubOAuthenticator.client_secret == 'something-else'
 
 
+def test_traefik_api_default():
+    """
+    Test default traefik api authentication settings with no overrides
+    """
+    c = apply_mock_config({})
+
+    assert c.TraefikTomlProxy.traefik_api_username == 'api_admin'
+    assert len(c.TraefikTomlProxy.traefik_api_password) == 0
+
+
+def test_set_traefik_api():
+    """
+    Test setting per traefik api credentials
+    """
+    c = apply_mock_config({
+        'traefik_api': {
+            'username': 'some_user',
+            'password': '1234'
+        }
+    })
+    assert c.TraefikTomlProxy.traefik_api_username == 'some_user'
+    assert c.TraefikTomlProxy.traefik_api_password == '1234'
+
+
+def test_load_secrets(tljh_dir):
+    """
+    Test loading secret files
+    """
+    with open(os.path.join(tljh_dir, 'state', 'traefik-api.secret'), 'w') as f:
+        f.write("traefik-password")
+
+    tljh_config = configurer.load_config()
+    assert tljh_config['traefik_api']['password'] == "traefik-password"
+    c = apply_mock_config(tljh_config)
+    assert c.TraefikTomlProxy.traefik_api_password == "traefik-password"
+
+
 def test_auth_native():
     """
     Test setting Native Authenticator
@@ -175,3 +214,4 @@ def test_auth_native():
     })
     assert c.JupyterHub.authenticator_class == 'nativeauthenticator.NativeAuthenticator'
     assert c.NativeAuthenticator.open_signup == True
+

tests/test_traefik.py

@@ -1,5 +1,6 @@
 """Test traefik configuration"""
 import os
+from unittest import mock
 
 import pytoml as toml
 
@@ -27,12 +28,19 @@ def test_default_config(tmpdir, tljh_dir):
         print(toml_cfg)
         cfg = toml.loads(toml_cfg)
     assert cfg["defaultEntryPoints"] == ["http"]
-    assert cfg["entryPoints"] == {"http": {"address": ":80"}}
-    assert cfg["frontends"] == {
-        "jupyterhub": {"backend": "jupyterhub", "passHostHeader": True}
-    }
-    assert cfg["backends"] == {
-        "jupyterhub": {"servers": {"chp": {"url": "http://127.0.0.1:15003"}}}
+    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
+    # runtime generated entry, value not testable
+    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
+
+    assert cfg["entryPoints"] == {
+        "http": {"address": ":80"},
+        "auth_api": {
+            "address": "127.0.0.1:8099",
+            "auth": {
+                "basic": {"users": [""]}
+            },
+            "whiteList": {"sourceRange": ["127.0.0.1"]}
+        },
     }
 
 
@@ -55,9 +63,20 @@ def test_letsencrypt_config(tljh_dir):
         cfg = toml.loads(toml_cfg)
     assert cfg["defaultEntryPoints"] == ["http", "https"]
     assert "acme" in cfg
+    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
+    # runtime generated entry, value not testable
+    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
+
     assert cfg["entryPoints"] == {
         "http": {"address": ":80", "redirect": {"entryPoint": "https"}},
-        "https": {"address": ":443", "backend": "jupyterhub", "tls": {}},
+        "https": {"address": ":443", "tls": {}},
+        "auth_api": {
+            "address": "127.0.0.1:8099",
+            "auth": {
+                "basic": {"users": [""]}
+            },
+            "whiteList": {"sourceRange": ["127.0.0.1"]}
+        },
     }
     assert cfg["acme"] == {
         "email": "fake@jupyter.org",
@@ -83,15 +102,24 @@ def test_manual_ssl_config(tljh_dir):
         cfg = toml.loads(toml_cfg)
     assert cfg["defaultEntryPoints"] == ["http", "https"]
     assert "acme" not in cfg
+    assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
+    # runtime generated entry, value not testable
+    cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
     assert cfg["entryPoints"] == {
         "http": {"address": ":80", "redirect": {"entryPoint": "https"}},
         "https": {
             "address": ":443",
-            "backend": "jupyterhub",
             "tls": {
                 "certificates": [
                     {"certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key"}
                 ]
             },
         },
+        "auth_api": {
+            "address": "127.0.0.1:8099",
+            "auth": {
+                "basic": {"users": [""]}
+            },
+            "whiteList": {"sourceRange": ["127.0.0.1"]}
+        },
     }

tljh/config.py

@@ -13,11 +13,15 @@
 """
 
 import argparse
+import asyncio
 from collections import Sequence, Mapping
 from copy import deepcopy
 import os
 import re
 import sys
+import time
+
+import requests
 
 from .yaml import yaml
 
@@ -85,7 +89,7 @@ def add_item_to_config(config, property_path, value):
 
 def remove_item_from_config(config, property_path, value):
     """
-    Add an item to a list in config.
+    Remove an item from a list in config.
     """
     path_components = property_path.split('.')
 
@@ -172,6 +176,12 @@ def remove_config_value(config_path, key_path, value):
     with open(config_path, 'w') as f:
         yaml.dump(config, f)
 
+def check_hub_ready():
+    try:
+        r = requests.get('http://127.0.0.1:80', verify=False)
+        return r.status_code == 200
+    except:
+        return False
 
 def reload_component(component):
     """
@@ -181,14 +191,20 @@ def reload_component(component):
     """
     # import here to avoid circular imports
     from tljh import systemd, traefik
+
     if component == 'hub':
         systemd.restart_service('jupyterhub')
-        # FIXME: Verify hub is back up?
+        # Ensure hub is back up
+        while not systemd.check_service_active('jupyterhub'):
+            time.sleep(1)
+        while not check_hub_ready():
+            time.sleep(1)
         print('Hub reload with new configuration complete')
     elif component == 'proxy':
         traefik.ensure_traefik_config(STATE_DIR)
-        systemd.restart_service('configurable-http-proxy')
         systemd.restart_service('traefik')
+        while not systemd.check_service_active('traefik'):
+            time.sleep(1)
         print('Proxy reload with new configuration complete')