diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 4dd1e4ec7..96b22bdfb 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -81,10 +81,7 @@ jobs: ${{ hashFiles('setup.py', 'dev-requirements.txt', '.github/workflows/unit-test.yaml') }} - name: Install Python dependencies - # Keep pip version pinning in sync with the one in bootstrap.py! - # See changelog at https://pip.pypa.io/en/latest/news/#changelog run: | - python3 -m pip install -U "pip==23.1.*" python3 -m pip install -r dev-requirements.txt python3 -m pip install -e . pip freeze diff --git a/MANIFEST.in b/MANIFEST.in index 4b590a36f..50c1cf761 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include tljh/systemd-units/* include tljh/*.tpl -include tljh/requirements-base.txt +include tljh/requirements-*.txt diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 9205f8fea..46acf3fca 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -475,11 +475,8 @@ def serve_forever(server): os.makedirs(hub_env_prefix, exist_ok=True) run_subprocess(["python3", "-m", "venv", hub_env_prefix]) - # Upgrade pip - # Keep pip version pinning in sync with the one in unit-test.yml! - # See changelog at https://pip.pypa.io/en/latest/news/#changelog logger.info("Upgrading pip...") - run_subprocess([hub_env_pip, "install", "--upgrade", "pip==23.1.*"]) + run_subprocess([hub_env_pip, "install", "--upgrade", "pip"]) # Install/upgrade TLJH installer tljh_install_cmd = [hub_env_pip, "install", "--upgrade"] diff --git a/tljh/installer.py b/tljh/installer.py index 755326ebe..668f30c46 100644 --- a/tljh/installer.py +++ b/tljh/installer.py @@ -106,25 +106,13 @@ def ensure_jupyterhub_package(prefix): hub environment be installed with pip prevents accidental mixing of python and conda packages! """ - # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically - # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 - # build-essential is also generally useful to everyone involved, and required for pycurl + # Install dependencies for installing pycurl via pip, where build-essential + # is generally useful for installing other packages as well. apt.install_packages(["libssl-dev", "libcurl4-openssl-dev", "build-essential"]) - conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True) - conda.ensure_pip_packages( + conda.ensure_pip_requirements( prefix, - [ - "jupyterhub==4.*", - "jupyterhub-systemdspawner==0.17.*", - "jupyterhub-firstuseauthenticator==1.*", - "jupyterhub-nativeauthenticator==1.*", - "jupyterhub-ldapauthenticator==1.*", - "jupyterhub-tmpauthenticator==0.6.*", - "oauthenticator==15.*", - "jupyterhub-idle-culler==1.*", - "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", - ], + os.path.join(HERE, "requirements-hub-env.txt"), upgrade=True, ) traefik.ensure_traefik_binary(prefix) @@ -158,9 +146,10 @@ def ensure_usergroups(): # minimum versions of packages MINIMUM_VERSIONS = { - # if conda/mamba are lower than this, upgrade them before installing the user packages + # if conda/mamba/pip are lower than this, upgrade them before installing the user packages "mamba": "0.16.0", "conda": "4.10", + "pip": "23.1.2", # minimum Python version (if not matched, abort to avoid big disruptive updates) "python": "3.9", } @@ -197,18 +186,16 @@ def ensure_user_environment(user_requirements_txt_file): # Check the existing environment for what to do package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX) + is_fresh_install = not package_versions - # Case 1: no existing environment - if not package_versions: - # 1a. no environment, but prefix exists. - # Abort to avoid clobbering something we don't recognize + if is_fresh_install: + # If no Python environment is detected but a folder exists we abort to + # avoid clobbering something we don't recognize. if os.path.exists(USER_ENV_PREFIX) and os.listdir(USER_ENV_PREFIX): msg = f"Found non-empty directory that is not a conda install in {USER_ENV_PREFIX}. Please remove it (or rename it to preserve files) and run tljh again." logger.error(msg) raise OSError(msg) - # 1b. No environment, directory empty or doesn't exist - # start fresh install logger.info("Downloading & setting up user environment...") installer_url, installer_sha256 = _mambaforge_url() with conda.download_miniconda_installer( @@ -216,11 +203,12 @@ def ensure_user_environment(user_requirements_txt_file): ) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) package_versions = conda.get_conda_package_versions(USER_ENV_PREFIX) + # quick sanity check: we should have conda and mamba! assert "conda" in package_versions assert "mamba" in package_versions - # next, check Python + # Check Python version python_version = package_versions["python"] logger.debug(f"Found python={python_version} in {USER_ENV_PREFIX}") if V(python_version) < V(MINIMUM_VERSIONS["python"]): @@ -232,10 +220,14 @@ def ensure_user_environment(user_requirements_txt_file): logger.error(msg) raise ValueError(msg) - # at this point, we know we have an env ready with conda and are going to start installing - # first, check if we should upgrade/install conda and/or mamba + # Ensure minimum versions of the following packages by upgrading to the + # latest if below that version. + # + # - conda/mamba, via conda-forge + # - pip, via PyPI + # to_upgrade = [] - for pkg in ("conda", "mamba"): + for pkg in ("conda", "mamba", "pip"): version = package_versions.get(pkg) min_version = MINIMUM_VERSIONS[pkg] if not version: @@ -249,20 +241,37 @@ def ensure_user_environment(user_requirements_txt_file): ) to_upgrade.append(pkg) - if to_upgrade: - conda.ensure_conda_packages( + cf_pkgs_to_upgrade = list(set(to_upgrade) & {"conda", "mamba"}) + if cf_pkgs_to_upgrade: + conda.ensure_conda_packages( + USER_ENV_PREFIX, + # we _could_ explicitly pin Python here, + # but conda already does this by default + cf_pkgs_to_upgrade, + ) + pypi_pkgs_to_upgrade = list(set(to_upgrade) & {"pip"}) + if pypi_pkgs_to_upgrade: + conda.ensure_pip_packages( + USER_ENV_PREFIX, + pypi_pkgs_to_upgrade, + upgrade=True, + ) + + # Install/upgrade the jupyterhub version in the user env based on the + # version specification used for the hub env. + # + with open(os.path.join(HERE, "requirements-hub-env.txt")) as f: + jh_version_spec = [l for l in f if l.startswith("jupyterhub>=")][0] + conda.ensure_pip_packages(USER_ENV_PREFIX, [jh_version_spec], upgrade=True) + + # Install user environment extras for initial installations + # + if is_fresh_install: + conda.ensure_pip_requirements( USER_ENV_PREFIX, - # we _could_ explicitly pin Python here, - # but conda already does this by default - to_upgrade, + os.path.join(HERE, "requirements-user-env-extras.txt"), ) - conda.ensure_pip_requirements( - USER_ENV_PREFIX, - os.path.join(HERE, "requirements-base.txt"), - upgrade=True, - ) - if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer conda.ensure_pip_requirements( diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt new file mode 100644 index 000000000..ba817672f --- /dev/null +++ b/tljh/requirements-hub-env.txt @@ -0,0 +1,29 @@ +# When tljh.installer runs, the hub' environment as typically found in +# /opt/tljh/hub, is upgraded to use these packages. +# +# When a new release is made, the lower bounds should be incremented to the +# latest release to help us narrow the versions based on knowing what tljh +# version is installed from inspecting this file. +# +# If a dependency is bumped to a new major version, we should make a major +# version release of tljh. +# +jupyterhub>=4.0.0,<5 +jupyterhub-systemdspawner>=0.17.0,<5 +jupyterhub-firstuseauthenticator>=1.0.0,<2 +jupyterhub-nativeauthenticator>=1.1.0,<2 +jupyterhub-ldapauthenticator>=1.3.2,<2 +jupyterhub-tmpauthenticator>=0.6.0,<0.7 +oauthenticator>=15.1.0,<16 +jupyterhub-idle-culler>=1.2.1,<2 +git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba + +# pycurl is installed to improve reliability and performance for when JupyterHub +# makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when +# making requests over tornado's SimpleHTTPClient automatically if pycurl is +# installed. +# +# ref: https://www.tornadoweb.org/en/stable/httpclient.html#module-tornado.simple_httpclient +# ref: https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 +# +pycurl>=7.45.2,<8 diff --git a/tljh/requirements-base.txt b/tljh/requirements-user-env-extras.txt similarity index 74% rename from tljh/requirements-base.txt rename to tljh/requirements-user-env-extras.txt index f926a5a55..7f27c7046 100644 --- a/tljh/requirements-base.txt +++ b/tljh/requirements-user-env-extras.txt @@ -1,14 +1,14 @@ # When tljh.installer runs, the users' environment as typically found in -# /opt/tljh/user, is setup with these packages. +# /opt/tljh/user, is installed with these packages. +# +# Whats listed here represents additional packages that the distributions +# installs initially, but doesn't upgrade as tljh is upgraded. # # WARNING: The order of these dependencies matters, this was observed when using # the requirements-txt-fixer pre-commit hook that sorted them and made # our integration tests fail. # -# JupyterHub + notebook package are base requirements for user environment -jupyterhub==4.* notebook==6.* -# Install additional notebook frontends! jupyterlab==3.* # nbgitpuller for easily pulling in Git repositories nbgitpuller==1.*