Let conda/pip update/upgrade packages

Since we now longer pin versions to the patch version, we should make
an install cause the packages upgrade within the version constraints
rathern than just settle for the current version if it is already
installed.

tljh/conda.py

@@ -93,7 +93,7 @@ def install_miniconda(installer_path, prefix):
     fix_permissions(prefix)
 
 
-def ensure_conda_packages(prefix, packages):
+def ensure_conda_packages(prefix, packages, update=False):
     """
     Ensure packages (from conda-forge) are installed in the conda prefix.
     """
@@ -103,12 +103,15 @@ def ensure_conda_packages(prefix, packages):
     # Explicitly do *not* capture stderr, since that's not always JSON!
     # Scripting conda is a PITA!
     # FIXME: raise different exception when using
+    update_flag = []
+    if update:
+        update_flag = ['--update-deps']
     raw_output = subprocess.check_output(conda_executable + [
         'install',
         '-c', 'conda-forge',  # Make customizable if we ever need to
         '--json',
         '--prefix', abspath
-    ] + packages).decode()
+    ] + update_flag + packages).decode()
     # `conda install` outputs JSON lines for fetch updates,
     # and a undelimited output at the end. There is no reasonable way to
     # parse this outside of this kludge.
@@ -124,32 +127,29 @@ def ensure_conda_packages(prefix, packages):
     fix_permissions(prefix)
 
 
-def ensure_pip_packages(prefix, packages):
+def ensure_pip_packages(prefix, packages, upgrade=False):
     """
     Ensure pip packages are installed in the given conda prefix.
     """
     abspath = os.path.abspath(prefix)
     pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
-
-    utils.run_subprocess(pip_executable + [
-        'install',
-        '--no-cache-dir',
-    ] + packages)
+    pip_cmd = pip_executable + ['install', '--no-cache-dir']
+    if upgrade:
+        pip_cmd.append('--upgrade')
+    utils.run_subprocess(pip_cmd + packages)
     fix_permissions(prefix)
 
 
-def ensure_pip_requirements(prefix, requirements_path):
+def ensure_pip_requirements(prefix, requirements_path, upgrade=False):
     """
     Ensure pip packages from given requirements_path are installed in given conda prefix.
 
     requirements_path can be a file or a URL.
     """
     abspath = os.path.abspath(prefix)
     pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
-
-    utils.run_subprocess(pip_executable + [
-        'install',
-        '-r',
-        requirements_path
-    ])
+    pip_cmd = pip_executable + ['install', '--no-cache-dir']
+    if upgrade:
+        pip_cmd.append('--upgrade')
+    utils.run_subprocess(pip_cmd + ['--requirement', requirements_path])
     fix_permissions(prefix)

tljh/installer.py

@@ -124,9 +124,7 @@ def ensure_jupyterhub_package(prefix):
         'libcurl4-openssl-dev',
         'build-essential'
     ])
-    conda.ensure_pip_packages(prefix, [
-        'pycurl==7.*'
-    ])
+    conda.ensure_pip_packages(prefix, ['pycurl==7.*'], upgrade=True)
 
     conda.ensure_pip_packages(
         prefix,
@@ -141,6 +139,7 @@ def ensure_jupyterhub_package(prefix):
             "jupyterhub-idle-culler==1.*",
             "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba"
         ],
+        upgrade=True,
     )
     traefik.ensure_traefik_binary(prefix)
 
@@ -193,20 +192,31 @@ def ensure_user_environment(user_requirements_txt_file):
             conda.install_miniconda(installer_path, USER_ENV_PREFIX)
         conda_version = '4.10.3'
 
-    conda.ensure_conda_packages(USER_ENV_PREFIX, [
-        # Conda's latest version is on conda much more so than on PyPI.
-        'conda==' + conda_version,
-        'mamba==' + mambaforge_mamba_version,
-    ])
+    conda.ensure_conda_packages(
+        USER_ENV_PREFIX,
+        [
+            # Conda's latest version is on conda much more so than on PyPI.
+            'conda==' + conda_version,
+            'mamba==' + mambaforge_mamba_version,
+        ],
+        # Updating isn't worth doing as long as we pin the dependency all the
+        # way to the patch version
+        update=False,
+    )
 
     conda.ensure_pip_requirements(
         USER_ENV_PREFIX,
         os.path.join(HERE, 'requirements-base.txt'),
+        upgrade=True,
     )
 
     if user_requirements_txt_file:
         # FIXME: This currently fails hard, should fail soft and not abort installer
-        conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
+        conda.ensure_pip_requirements(
+            USER_ENV_PREFIX,
+            user_requirements_txt_file,
+            upgrade=True,
+        )
 
 
 def ensure_admins(admin_password_list):
@@ -313,7 +323,7 @@ def setup_plugins(plugins=None):
     """
     # Install plugins
     if plugins:
-        conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins)
+        conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins, upgrade=True)
 
     # Set up plugin infrastructure
     pm = pluggy.PluginManager('tljh')
@@ -342,23 +352,35 @@ def run_plugin_actions(plugin_manager):
         logger.info('Installing {} hub pip packages collected from plugins: {}'.format(
             len(hub_pip_packages), ' '.join(hub_pip_packages)
         ))
-        conda.ensure_pip_packages(HUB_ENV_PREFIX, hub_pip_packages)
+        conda.ensure_pip_packages(
+            HUB_ENV_PREFIX,
+            hub_pip_packages,
+            upgrade=True,
+        )
 
     # Install conda packages
     conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages())))
     if conda_packages:
         logger.info('Installing {} user conda packages collected from plugins: {}'.format(
             len(conda_packages), ' '.join(conda_packages)
         ))
-        conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages)
+        conda.ensure_conda_packages(
+            USER_ENV_PREFIX,
+            conda_packages,
+            update=True,
+        )
 
     # Install pip packages
     user_pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages())))
     if user_pip_packages:
         logger.info('Installing {} user pip packages collected from plugins: {}'.format(
             len(user_pip_packages), ' '.join(user_pip_packages)
         ))
-        conda.ensure_pip_packages(USER_ENV_PREFIX, user_pip_packages)
+        conda.ensure_pip_packages(
+            USER_ENV_PREFIX,
+            user_pip_packages,
+            upgrade=True,
+        )
 
     # Custom post install actions
     hook.tljh_post_install()