Merge pull request #719 from consideRatio/pr/update-hub-user-env-and-…

…tljh-deps

Bump to recent versions, and make bootstrap.py update to those when run

dev-requirements.txt

@@ -2,4 +2,3 @@ pytest
 pytest-cov
 pytest-mock
 codecov
-chardet==3.0.4

docs/howto/auth/dummy.rst

@@ -26,7 +26,7 @@ Enabling the authenticator
 
    .. code-block:: bash
 
-      sudo tljh-config set auth.type dummyauthenticator.DummyAuthenticator
+      sudo tljh-config set auth.type dummy
 
    .. code-block:: bash
 

integration-tests/test_hub.py

@@ -34,7 +34,7 @@ async def test_user_code_execute():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
     async with User(username, hub_url, partial(login_dummy, password='')) as u:
@@ -58,7 +58,7 @@ async def test_user_server_started_with_custom_base_url():
     hub_url = f"http://localhost{base_url}"
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'base_url', base_url)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
@@ -81,7 +81,7 @@ async def test_user_admin_add():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
@@ -111,7 +111,7 @@ async def test_user_admin_remove():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
@@ -147,7 +147,7 @@ async def test_long_username():
     hub_url = 'http://localhost'
     username = secrets.token_hex(32)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
     try:
@@ -183,7 +183,7 @@ async def test_user_group_adding():
     # Create the group we want to add the user to
     system('groupadd somegroup')
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.extra_user_groups.somegroup', username)).wait()
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()
 
@@ -223,7 +223,7 @@ async def test_idle_server_culled():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     # Check every 10s for idle servers to cull
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
     # Apart from servers, also cull users
@@ -269,7 +269,7 @@ async def test_active_server_not_culled():
     hub_url = 'http://localhost'
     username = secrets.token_hex(8)
 
-    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
+    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummy')).wait()
     # Check every 10s for idle servers to cull
     assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'services.cull.every', "10")).wait()
     # Apart from servers, also cull users

setup.py

@@ -11,9 +11,9 @@
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
-        'ruamel.yaml==0.15.*',
+        'ruamel.yaml==0.17.*',
         'jinja2',
-        'pluggy>0.7<1.0',
+        'pluggy==1.*',
         'passlib',
         'backoff',
         'requests',

tests/test_configurer.py

@@ -98,13 +98,13 @@ def test_auth_dummy():
     """
     c = apply_mock_config({
         'auth': {
-            'type': 'dummyauthenticator.DummyAuthenticator',
+            'type': 'dummy',
             'DummyAuthenticator': {
                 'password': 'test'
             }
         }
     })
-    assert c.JupyterHub.authenticator_class == 'dummyauthenticator.DummyAuthenticator'
+    assert c.JupyterHub.authenticator_class == 'dummy'
     assert c.DummyAuthenticator.password == 'test'
 
 

tljh/conda.py

@@ -96,6 +96,9 @@ def install_miniconda(installer_path, prefix):
 def ensure_conda_packages(prefix, packages):
     """
     Ensure packages (from conda-forge) are installed in the conda prefix.
+
+    Note that conda seem to update dependencies by default, so there is probably
+    no need to have a update parameter exposed for this function.
     """
     conda_executable = [os.path.join(prefix, 'bin', 'mamba')]
     abspath = os.path.abspath(prefix)
@@ -124,32 +127,29 @@ def ensure_conda_packages(prefix, packages):
     fix_permissions(prefix)
 
 
-def ensure_pip_packages(prefix, packages):
+def ensure_pip_packages(prefix, packages, upgrade=False):
     """
     Ensure pip packages are installed in the given conda prefix.
     """
     abspath = os.path.abspath(prefix)
     pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
-
-    utils.run_subprocess(pip_executable + [
-        'install',
-        '--no-cache-dir',
-    ] + packages)
+    pip_cmd = pip_executable + ['install']
+    if upgrade:
+        pip_cmd.append('--upgrade')
+    utils.run_subprocess(pip_cmd + packages)
     fix_permissions(prefix)
 
 
-def ensure_pip_requirements(prefix, requirements_path):
+def ensure_pip_requirements(prefix, requirements_path, upgrade=False):
     """
     Ensure pip packages from given requirements_path are installed in given conda prefix.
 
     requirements_path can be a file or a URL.
     """
     abspath = os.path.abspath(prefix)
     pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
-
-    utils.run_subprocess(pip_executable + [
-        'install',
-        '-r',
-        requirements_path
-    ])
+    pip_cmd = pip_executable + ['install']
+    if upgrade:
+        pip_cmd.append('--upgrade')
+    utils.run_subprocess(pip_cmd + ['--requirement', requirements_path])
     fix_permissions(prefix)

tljh/installer.py

@@ -124,25 +124,22 @@ def ensure_jupyterhub_package(prefix):
         'libcurl4-openssl-dev',
         'build-essential'
     ])
-    conda.ensure_pip_packages(prefix, [
-        'pycurl==7.43.*'
-    ])
+    conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True)
 
     conda.ensure_pip_packages(
         prefix,
         [
-            "jupyterhub==1.4.0",
-            "jupyterhub-dummyauthenticator==0.3.1",
-            "jupyterhub-systemdspawner==0.15",
-            "jupyterhub-firstuseauthenticator==0.14.1",
-            "jupyterhub-nativeauthenticator==0.0.7",
-            "jupyterhub-ldapauthenticator==1.3.0",
-            "jupyterhub-tmpauthenticator==0.6",
-            "oauthenticator==0.10.0",
-            "jupyterhub-idle-culler==1.0",
-            "chardet==3.0.4",
+            "jupyterhub==1.*",
+            "jupyterhub-systemdspawner==0.15.*",
+            "jupyterhub-firstuseauthenticator==0.14.*",
+            "jupyterhub-nativeauthenticator==1.*",
+            "jupyterhub-ldapauthenticator==1.*",
+            "jupyterhub-tmpauthenticator==0.6.*",
+            "oauthenticator==14.*",
+            "jupyterhub-idle-culler==1.*",
             "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba"
         ],
+        upgrade=True,
     )
     traefik.ensure_traefik_binary(prefix)
 
@@ -195,20 +192,28 @@ def ensure_user_environment(user_requirements_txt_file):
             conda.install_miniconda(installer_path, USER_ENV_PREFIX)
         conda_version = '4.10.3'
 
-    conda.ensure_conda_packages(USER_ENV_PREFIX, [
-        # Conda's latest version is on conda much more so than on PyPI.
-        'conda==' + conda_version,
-        'mamba==' + mambaforge_mamba_version,
-    ])
+    conda.ensure_conda_packages(
+        USER_ENV_PREFIX,
+        [
+            # Conda's latest version is on conda much more so than on PyPI.
+            'conda==' + conda_version,
+            'mamba==' + mambaforge_mamba_version,
+        ],
+    )
 
     conda.ensure_pip_requirements(
         USER_ENV_PREFIX,
         os.path.join(HERE, 'requirements-base.txt'),
+        upgrade=True,
     )
 
     if user_requirements_txt_file:
         # FIXME: This currently fails hard, should fail soft and not abort installer
-        conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
+        conda.ensure_pip_requirements(
+            USER_ENV_PREFIX,
+            user_requirements_txt_file,
+            upgrade=True,
+        )
 
 
 def ensure_admins(admin_password_list):
@@ -315,7 +320,7 @@ def setup_plugins(plugins=None):
     """
     # Install plugins
     if plugins:
-        conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins)
+        conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins, upgrade=True)
 
     # Set up plugin infrastructure
     pm = pluggy.PluginManager('tljh')
@@ -344,7 +349,11 @@ def run_plugin_actions(plugin_manager):
         logger.info('Installing {} hub pip packages collected from plugins: {}'.format(
             len(hub_pip_packages), ' '.join(hub_pip_packages)
         ))
-        conda.ensure_pip_packages(HUB_ENV_PREFIX, hub_pip_packages)
+        conda.ensure_pip_packages(
+            HUB_ENV_PREFIX,
+            hub_pip_packages,
+            upgrade=True,
+        )
 
     # Install conda packages
     conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages())))
@@ -360,7 +369,11 @@ def run_plugin_actions(plugin_manager):
         logger.info('Installing {} user pip packages collected from plugins: {}'.format(
             len(user_pip_packages), ' '.join(user_pip_packages)
         ))
-        conda.ensure_pip_packages(USER_ENV_PREFIX, user_pip_packages)
+        conda.ensure_pip_packages(
+            USER_ENV_PREFIX,
+            user_pip_packages,
+            upgrade=True,
+        )
 
     # Custom post install actions
     hook.tljh_post_install()

tljh/requirements-base.txt

@@ -1,18 +1,21 @@
+# When tljh.installer runs, the users' environment as typically found in
+# /opt/tljh/user, is setup with these packages.
+#
 # FIXME: a frozen version of this file should be used
 # pinning only direct dependencies is a recipe for broken environments!
 
 # JupyterHub + notebook package are base requirements for user environment
-jupyterhub==1.4.0
-notebook==6.4.1
+jupyterhub==1.*
+notebook==6.*
 # Install additional notebook frontends!
 jupyterlab==3.*
-nteract-on-jupyter==2.1.*
+nteract-on-jupyter==2.*
 # Install jupyterlab extensions from PyPI
 # nbgitpuller for easily pulling in Git repositories
 nbgitpuller==1.*
 # jupyter-resource-usage to show people how much RAM they are using
-jupyter-resource-usage==0.5.*
+jupyter-resource-usage==0.6.*
 # Most people consider ipywidgets to be part of the core notebook experience
-ipywidgets==7.6.*
+ipywidgets==7.*
 # Pin tornado
 tornado>=6.1