From a86e4ce153c452501f5a1d84015c6a3d078f7a28 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 13 May 2023 22:25:20 +0200 Subject: [PATCH 1/6] user env: only upgrade jupyterhub in user env when upgrading tljh --- MANIFEST.in | 2 +- tljh/installer.py | 12 +++++++++++- ...nts-base.txt => requirements-user-env-extras.txt} | 8 ++++---- tljh/requirements-user-env.txt | 7 +++++++ 4 files changed, 23 insertions(+), 6 deletions(-) rename tljh/{requirements-base.txt => requirements-user-env-extras.txt} (74%) create mode 100644 tljh/requirements-user-env.txt diff --git a/MANIFEST.in b/MANIFEST.in index 4b590a36f..50c1cf761 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include tljh/systemd-units/* include tljh/*.tpl -include tljh/requirements-base.txt +include tljh/requirements-*.txt diff --git a/tljh/installer.py b/tljh/installer.py index 755326ebe..0054ab9ad 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -200,6 +200,8 @@ def ensure_user_environment(user_requirements_txt_file): # Case 1: no existing environment if not package_versions: + is_fresh_install = True + # 1a. no environment, but prefix exists. # Abort to avoid clobbering something we don't recognize if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX): @@ -219,6 +221,8 @@ def ensure_user_environment(user_requirements_txt_file): # quick sanity check: we should have conda and mamba! assert "conda" in package_versions assert "mamba" in package_versions + else: + is_fresh_install = False # next, check Python python_version = package_versions["python"] @@ -259,9 +263,15 @@ def ensure_user_environment(user_requirements_txt_file): conda.ensure_pip_requirements( USER_ENV_PREFIX, - os.path.join(HERE, "requirements-base.txt"), + os.path.join(HERE, "requirements-user-env.txt"), upgrade=True, ) + if is_fresh_install: + conda.ensure_pip_requirements( + USER_ENV_PREFIX, + os.path.join(HERE, "requirements-user-env-extras.txt"), + upgrade=True, + ) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer diff --git a/tljh/requirements-base.txt b/tljh/requirements-user-env-extras.txt similarity index 74% rename from tljh/requirements-base.txt rename to tljh/requirements-user-env-extras.txt index f926a5a55..7f27c7046 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-user-env-extras.txt @@ -1,14 +1,14 @@ # When tljh.installer runs, the users' environment as typically found in -# /opt/tljh/user, is setup with these packages. +# /opt/tljh/user, is installed with these packages. +# +# Whats listed here represents additional packages that the distributions +# installs initially, but doesn't upgrade as tljh is upgraded. # # WARNING: The order of these dependencies matters, this was observed when using # the requirements-txt-fixer pre-commit hook that sorted them and made # our integration tests fail. # -# JupyterHub + notebook package are base requirements for user environment -jupyterhub==4.* notebook==6.* -# Install additional notebook frontends! jupyterlab==3.* # nbgitpuller for easily pulling in Git repositories nbgitpuller==1.* diff --git a/tljh/requirements-user-env.txt b/tljh/requirements-user-env.txt new file mode 100644 index 000000000..a9528e90f --- /dev/null +++ b/tljh/requirements-user-env.txt @@ -0,0 +1,7 @@ +# When tljh.installer runs, the users' environment as typically found in +# /opt/tljh/user, is installed or upgraded with these packages. +# +# Whats listed here should be only the unconditional requirements that we have in +# the user environment. +# +jupyterhub==4.* From eaa16babb543458e77d5558f5990c9a80a6d3975 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 23:42:53 +0200 Subject: [PATCH 2/6] hub env: use req. file and add lower bounds for readability --- tljh/installer.py | 20 ++++---------------- tljh/requirements-hub-env.txt | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 tljh/requirements-hub-env.txt diff --git a/tljh/installer.py b/tljh/installer.py index 0054ab9ad..fc968a398 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -106,25 +106,13 @@ def ensure_jupyterhub_package(prefix): hub environment be installed with pip prevents accidental mixing of python and conda packages! """ - # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically - # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 - # build-essential is also generally useful to everyone involved, and required for pycurl + # Install dependencies for installing pycurl via pip, where build-essential + # is generally useful for installing other packages as well. apt.install_packages(["libssl-dev", "libcurl4-openssl-dev", "build-essential"]) - conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True) - conda.ensure_pip_packages( + conda.ensure_pip_requirements( prefix, - [ - "jupyterhub==4.*", - "jupyterhub-systemdspawner==0.17.*", - "jupyterhub-firstuseauthenticator==1.*", - "jupyterhub-nativeauthenticator==1.*", - "jupyterhub-ldapauthenticator==1.*", - "jupyterhub-tmpauthenticator==0.6.*", - "oauthenticator==15.*", - "jupyterhub-idle-culler==1.*", - "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", - ], + os.path.join(HERE, "requirements-hub-env.txt"), upgrade=True, ) traefik.ensure_traefik_binary(prefix) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt new file mode 100644 index 000000000..ba817672f --- /dev/null +++ b/tljh/requirements-hub-env.txt @@ -0,0 +1,29 @@ +# When tljh.installer runs, the hub' environment as typically found in +# /opt/tljh/hub, is upgraded to use these packages. +# +# When a new release is made, the lower bounds should be incremented to the +# latest release to help us narrow the versions based on knowing what tljh +# version is installed from inspecting this file. +# +# If a dependency is bumped to a new major version, we should make a major +# version release of tljh. +# +jupyterhub>=4.0.0,<5 +jupyterhub-systemdspawner>=0.17.0,<5 +jupyterhub-firstuseauthenticator>=1.0.0,<2 +jupyterhub-nativeauthenticator>=1.1.0,<2 +jupyterhub-ldapauthenticator>=1.3.2,<2 +jupyterhub-tmpauthenticator>=0.6.0,<0.7 +oauthenticator>=15.1.0,<16 +jupyterhub-idle-culler>=1.2.1,<2 +git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba + +# pycurl is installed to improve reliability and performance for when JupyterHub +# makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when +# making requests over tornado's SimpleHTTPClient automatically if pycurl is +# installed. +# +# ref: https://www.tornadoweb.org/en/stable/httpclient.html#module-tornado.simple_httpclient +# ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 +# +pycurl>=7.45.2,<8 From 9c83e9000e6a70acbd1e62987944e00eedb7652a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 23:45:27 +0200 Subject: [PATCH 3/6] user env: ensure pip>=23.1.2 on install --- tljh/installer.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index fc968a398..f94a63f40 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -146,9 +146,10 @@ def ensure_usergroups(): # minimum versions of packages MINIMUM_VERSIONS = { - # if conda/mamba are lower than this, upgrade them before installing the user packages + # if conda/mamba/pip are lower than this, upgrade them before installing the user packages "mamba": "0.16.0", "conda": "4.10", + "pip": "23.1.2", # minimum Python version (if not matched, abort to avoid big disruptive updates) "python": "3.9", } @@ -224,10 +225,14 @@ def ensure_user_environment(user_requirements_txt_file): logger.error(msg) raise ValueError(msg) - # at this point, we know we have an env ready with conda and are going to start installing - # first, check if we should upgrade/install conda and/or mamba + # Ensure minimum versions of the following packages by upgrading to the + # latest if below that version. + # + # - conda/mamba, via conda-forge + # - pip, via PyPI + # to_upgrade = [] - for pkg in ("conda", "mamba"): + for pkg in ("conda", "mamba", "pip"): version = package_versions.get(pkg) min_version = MINIMUM_VERSIONS[pkg] if not version: @@ -241,19 +246,21 @@ def ensure_user_environment(user_requirements_txt_file): ) to_upgrade.append(pkg) - if to_upgrade: - conda.ensure_conda_packages( - USER_ENV_PREFIX, - # we _could_ explicitly pin Python here, - # but conda already does this by default - to_upgrade, - ) - - conda.ensure_pip_requirements( - USER_ENV_PREFIX, - os.path.join(HERE, "requirements-user-env.txt"), - upgrade=True, - ) + cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"}) + if cf_pkgs_to_upgrade: + conda.ensure_conda_packages( + USER_ENV_PREFIX, + # we _could_ explicitly pin Python here, + # but conda already does this by default + cf_pkgs_to_upgrade, + ) + pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"}) + if pypi_pkgs_to_upgrade: + conda.ensure_pip_packages( + USER_ENV_PREFIX, + pypi_pkgs_to_upgrade, + upgrade=True, + ) if is_fresh_install: conda.ensure_pip_requirements( USER_ENV_PREFIX, From 6388d390e148366c98e0a43e93ceb37458394487 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 23:47:04 +0200 Subject: [PATCH 4/6] refactor: improve readability of a section --- tljh/installer.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tljh/installer.py b/tljh/installer.py index f94a63f40..6187d8525 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -186,20 +186,16 @@ def ensure_user_environment(user_requirements_txt_file): # Check the existing environment for what to do package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX) + is_fresh_install = not package_versions - # Case 1: no existing environment - if not package_versions: - is_fresh_install = True - - # 1a. no environment, but prefix exists. - # Abort to avoid clobbering something we don't recognize + if is_fresh_install: + # If no Python environment is detected but a folder exists we abort to + # avoid clobbering something we don't recognize. if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX): msg = f"Found non-empty directory that is not a conda install in {USER_ENV_PREFIX}. Please remove it (or rename it to preserve files) and run tljh again." logger.error(msg) raise OSError(msg) - # 1b. No environment, directory empty or doesn't exist - # start fresh install logger.info("Downloading & setting up user environment...") installer_url, installer_sha256 = _mambaforge_url() with conda.download_miniconda_installer( @@ -207,13 +203,12 @@ def ensure_user_environment(user_requirements_txt_file): ) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX) + # quick sanity check: we should have conda and mamba! assert "conda" in package_versions assert "mamba" in package_versions - else: - is_fresh_install = False - # next, check Python + # Check Python version python_version = package_versions["python"] logger.debug(f"Found python={python_version} in {USER_ENV_PREFIX}") if V(python_version) < V(MINIMUM_VERSIONS["python"]): From c1aa30a479c6b03c70ae41c9848383560f1abbce Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 23:49:39 +0200 Subject: [PATCH 5/6] hub env: remove pip pinning, always upgrade pip --- .github/workflows/unit-test.yaml | 3 --- bootstrap/bootstrap.py | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 4dd1e4ec7..96b22bdfb 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -81,10 +81,7 @@ jobs: ${{ hashFiles('setup.py', 'dev-requirements.txt', '.github/workflows/unit-test.yaml') }} - name: Install Python dependencies - # Keep pip version pinning in sync with the one in bootstrap.py! - # See changelog at https://pip.pypa.io/en/latest/news/#changelog run: | - python3 -m pip install -U "pip==23.1.*" python3 -m pip install -r dev-requirements.txt python3 -m pip install -e . pip freeze diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 9205f8fea..46acf3fca 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -475,11 +475,8 @@ def serve_forever(server): os.makedirs(hub_env_prefix, exist_ok=True) run_subprocess(["python3", "-m", "venv", hub_env_prefix]) - # Upgrade pip - # Keep pip version pinning in sync with the one in unit-test.yml! - # See changelog at https://pip.pypa.io/en/latest/news/#changelog logger.info("Upgrading pip...") - run_subprocess([hub_env_pip, "install", "--upgrade", "pip==23.1.*"]) + run_subprocess([hub_env_pip, "install", "--upgrade", "pip"]) # Install/upgrade TLJH installer tljh_install_cmd = [hub_env_pip, "install", "--upgrade"] From 21312b2cfd1f890809409e1de5b91570221c8b3a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 18 May 2023 23:51:00 +0200 Subject: [PATCH 6/6] user env: upgrade jh based on hub spec --- tljh/installer.py | 11 ++++++++++- tljh/requirements-user-env.txt | 7 ------- 2 files changed, 10 insertions(+), 8 deletions(-) delete mode 100644 tljh/requirements-user-env.txt diff --git a/tljh/installer.py b/tljh/installer.py index 6187d8525..668f30c46 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -256,11 +256,20 @@ def ensure_user_environment(user_requirements_txt_file): pypi_pkgs_to_upgrade, upgrade=True, ) + + # Install/upgrade the jupyterhub version in the user env based on the + # version specification used for the hub env. + # + with open(os.path.join(HERE, "requirements-hub-env.txt")) as f: + jh_version_spec = [l for l in f if l.startswith("jupyterhub>=")][0] + conda.ensure_pip_packages(USER_ENV_PREFIX, [jh_version_spec], upgrade=True) + + # Install user environment extras for initial installations + # if is_fresh_install: conda.ensure_pip_requirements( USER_ENV_PREFIX, os.path.join(HERE, "requirements-user-env-extras.txt"), - upgrade=True, ) if user_requirements_txt_file: diff --git a/tljh/requirements-user-env.txt b/tljh/requirements-user-env.txt deleted file mode 100644 index a9528e90f..000000000 --- a/tljh/requirements-user-env.txt +++ /dev/null @@ -1,7 +0,0 @@ -# When tljh.installer runs, the users' environment as typically found in -# /opt/tljh/user, is installed or upgraded with these packages. -# -# Whats listed here should be only the unconditional requirements that we have in -# the user environment. -# -jupyterhub==4.*