chore: remove support for Python 3.7; prefer 3.8 (#7329)

Python 3.7 is EOL. * Some docs that referred to 3.7 now refer to 3.8 * Training API "how to" doc no longer mentions any 3.7 docker images * `CONTRIBUTING.md` no longer says to upgrade pip when installing determined * `CONTRIBUTING.md` has revised text about Python requirements * mypy tests against 3.8. * Features new to 3.8 can be used without suppressing type checking or conditional imports. These changes have been made. * No longer run `test-cli` in CircleCI against Python 3.7. * There are no wheels for TF1 using Python 3.8. Removing 3.7 support means TF1 cannot be used. In previous commits, we removed support of TF1. We also clean up those references here. In particular: - `e2e_tests/tests/command/test_run:test_k8_init_containers` and `e2e_tests/tests/command/test_run:test_k8_sidecars` now run on TF2 images - Removed references to TF1 images in `e2e_tests`. - Removed references to TF1 Docker images in `bumpenvs.yaml`
determined-ai · Mar 4, 2024 · fa856ab · fa856ab
1 parent c404c8e
commit fa856ab
Show file tree

Hide file tree

Showing 15 changed files with 27 additions and 82 deletions.
diff --git a/.circleci/real_config.yml b/.circleci/real_config.yml
@@ -12,9 +12,6 @@ orbs:
   win: circleci/windows@5.0.0
 
 executors:
-  python-37:
-    docker:
-      - image: python:3.7-slim-buster
   python-38:
     docker:
       - image: python:3.8-slim-buster
@@ -2746,7 +2743,7 @@ workflows:
           matrix:
             parameters:
               executor-name:
-                ["python-37", "python-38", "python-39", "win/default"]
+                ["python-38", "python-39", "win/default"]
 
   test-unit:
     jobs:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -46,9 +46,8 @@ git clone --recurse-submodules https://github.com/determined-ai/determined.git
 
 #### Prerequisites
 
-- Go (>= 1.20)
-- Python (>= 3.7.4, <= 3.9), including:
-  - python3-venv
+- Go (>= 1.22)
+- Python 3.8. Newer Python versions are partially supported, but some components may not work (e.g., docs build).
   - python3-wheel
   - python3-dev
 - Node (>= 20.9.0, < 21)
@@ -118,7 +117,6 @@ source $HOME/.profile
 cd determined
 python3 -m venv $HOME/.virtualenvs/determined
 . $HOME/.virtualenvs/determined/bin/activate
-$HOME/.virtualenvs/determined/bin/python3.9 -m pip install --upgrade pip
 export PATH=$PATH:$HOME/go/bin
 make all
 ```

diff --git a/docs/model-dev-guide/api-guides/apis-howto/_index.rst b/docs/model-dev-guide/api-guides/apis-howto/_index.rst
@@ -83,7 +83,6 @@ respectively:
 We also provide lightweight CPU-only counterparts:
 
 -  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.27.1``
--  ``determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-0.21.2``
 -  ``determinedai/environments:py-3.8-tf-2.8-cpu-0.27.1``
 
 To change the container image used for an experiment, specify :ref:`environment.image

diff --git a/docs/tools/cli/cli-ug.rst b/docs/tools/cli/cli-ug.rst
@@ -35,9 +35,10 @@ and upgrade.
  Installation
 **************
 
-The CLI is distributed as a Python wheel package and requires Python >= 3.7. We recommend setting up
-a `virtualenv <https://virtualenv.pypa.io/en/latest/>`__ and using the ``pip`` utility to install
-``determined`` into the environment:
+The command-line interface (CLI) is distributed in the form of a Python wheel package and requires
+Python version 3.8 or later. We recommend setting up a `virtualenv
+<https://virtualenv.pypa.io/en/latest/>`__ and using the ``pip`` utility to install ``determined``
+into the environment:
 
 .. code::
 

diff --git a/e2e_tests/mypy.ini b/e2e_tests/mypy.ini
@@ -1,6 +1,6 @@
 [mypy]
 mypy_path = ../harness
-python_version = 3.7
+python_version = 3.8
 follow_imports = silent
 ignore_missing_imports = True
 

diff --git a/e2e_tests/tests/command/test_run.py b/e2e_tests/tests/command/test_run.py
@@ -286,7 +286,7 @@ def test_k8s_init_containers() -> None:
                 "spec": {
                     "initContainers": [
                         {
-                            "image": conf.TF1_CPU_IMAGE,
+                            "image": conf.TF2_CPU_IMAGE,
                             "name": "simple-init-container",
                             "command": ["/bin/bash"],
                             "args": ["-c", "exit 1"],
@@ -313,7 +313,7 @@ def test_k8s_sidecars() -> None:
                 "spec": {
                     "containers": [
                         {
-                            "image": conf.TF1_CPU_IMAGE,
+                            "image": conf.TF2_CPU_IMAGE,
                             "name": "sidecar",
                             "command": ["/bin/bash"],
                         }

diff --git a/e2e_tests/tests/config.py b/e2e_tests/tests/config.py
@@ -14,18 +14,14 @@
 MAX_TRIAL_BUILD_SECS = 90
 
 
-DEFAULT_TF1_CPU_IMAGE = "determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-6eceaca"
 DEFAULT_TF2_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-f66cbce"
-DEFAULT_TF1_GPU_IMAGE = "determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-6eceaca"
 DEFAULT_TF2_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-f66cbce"
 DEFAULT_PT_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-cpu-f66cbce"
 DEFAULT_PT_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-gpu-f66cbce"
 DEFAULT_PT2_CPU_IMAGE = "determinedai/environments:py-3.10-pytorch-2.0-cpu-f66cbce"
 DEFAULT_PT2_GPU_IMAGE = "determinedai/environments:cuda-11.8-pytorch-2.0-gpu-f66cbce"
 
-TF1_CPU_IMAGE = os.environ.get("TF1_CPU_IMAGE") or DEFAULT_TF1_CPU_IMAGE
 TF2_CPU_IMAGE = os.environ.get("TF2_CPU_IMAGE") or DEFAULT_TF2_CPU_IMAGE
-TF1_GPU_IMAGE = os.environ.get("TF1_GPU_IMAGE") or DEFAULT_TF1_GPU_IMAGE
 TF2_GPU_IMAGE = os.environ.get("TF2_GPU_IMAGE") or DEFAULT_TF2_GPU_IMAGE
 PT_CPU_IMAGE = os.environ.get("PT_CPU_IMAGE") or DEFAULT_PT_CPU_IMAGE
 PT_GPU_IMAGE = os.environ.get("PT_GPU_IMAGE") or DEFAULT_PT_GPU_IMAGE
@@ -143,10 +139,6 @@ def set_image(config: Dict[Any, Any], cpu_image: str, gpu_image: str) -> Dict[An
     return config
 
 
-def set_tf1_image(config: Dict[Any, Any]) -> Dict[Any, Any]:
-    return set_image(config, TF1_CPU_IMAGE, TF1_GPU_IMAGE)
-
-
 def set_tf2_image(config: Dict[Any, Any]) -> Dict[Any, Any]:
     return set_image(config, TF2_CPU_IMAGE, TF2_GPU_IMAGE)
 

diff --git a/e2e_tests/tests/model_hub/test_mmdetection.py b/e2e_tests/tests/model_hub/test_mmdetection.py
@@ -15,7 +15,7 @@ def set_docker_image(config: Dict) -> Dict:
     )
 
     config = conf.set_image(
-        config, conf.TF1_CPU_IMAGE, f"determinedai/model-hub-mmdetection:{git_short_hash}"
+        config, conf.TF2_CPU_IMAGE, f"determinedai/model-hub-mmdetection:{git_short_hash}"
     )
     return config
 

diff --git a/harness/determined/cli/dev.py b/harness/determined/cli/dev.py
@@ -7,9 +7,10 @@
 import shutil
 import subprocess
 import sys
+import typing
 from argparse import Namespace
 from collections.abc import Sequence
-from typing import Any, Dict, List, OrderedDict, Tuple
+from typing import Any, Dict, List, Optional, OrderedDict, Tuple, Union, get_args, get_origin
 from urllib import parse
 
 from termcolor import colored
@@ -53,10 +54,7 @@ def curl(args: Namespace) -> None:
         cmd += args.curl_args
 
     if args.x:
-        if hasattr(shlex, "join"):  # added in py 3.8
-            print(shlex.join(cmd))  # type: ignore
-        else:
-            print(" ".join(shlex.quote(arg) for arg in cmd))
+        print(shlex.join(cmd))
 
     if not sys.stdout.isatty():
         output = subprocess.run(cmd)
@@ -99,20 +97,13 @@ def unwrap_optional(annotation: Any) -> Any:
     """
     evaluates and unwraps a typing.Optional annotation to its inner type.
     """
-    import typing
-
-    try:
-        from typing import get_args, get_origin  # type: ignore
-    except ImportError:
-        raise errors.CliError("python >= 3.8 is required to use this feature")
-
     local_context = {
         "typing": typing,
-        "Optional": typing.Optional,
-        "Union": typing.Union,
-        "List": typing.List,
-        "Sequence": typing.Sequence,
-        "Dict": typing.Dict,
+        "Optional": Optional,
+        "Union": Union,
+        "List": List,
+        "Sequence": Sequence,
+        "Dict": Dict,
         "NoneType": type(None),
     }
     if isinstance(annotation, str):
@@ -123,10 +114,10 @@ def unwrap_optional(annotation: Any) -> Any:
     origin = get_origin(annotation)
     args = get_args(annotation)
 
-    if origin is typing.Union:
+    if origin is Union:
         if len(args) == 2 and type(None) in args:
             return args[0]
-    elif origin is typing.Optional:
+    elif origin is Optional:
         return args[0]
     return annotation
 
@@ -136,11 +127,6 @@ def is_supported_annotation(annot: Any) -> bool:
     determines if a our CLI deserializer supports a given type annotation
     and subsequently a binding's parameter.
     """
-    try:
-        from typing import get_args, get_origin  # type: ignore
-    except ImportError:
-        raise errors.CliError("python >= 3.8 is required to use this feature")
-
     annot = unwrap_optional(annot)
     supported_types = [str, int, float, type(None), bool]
     if annot in supported_types:

diff --git a/harness/determined/common/experimental/checkpoint/_checkpoint.py b/harness/determined/common/experimental/checkpoint/_checkpoint.py
@@ -4,7 +4,6 @@
 import logging
 import pathlib
 import shutil
-import sys
 import tarfile
 import warnings
 from typing import Any, Dict, Iterable, List, Optional
@@ -276,23 +275,12 @@ def _download_auto(
                     "but they both failed."
                 ) from e
 
-    def _shutil_copytree(self, src: str, dst: str) -> None:
-        # TODO: remove version check once we drop support for Python 3.7
-        if sys.version_info.minor >= 8:
-            shutil.copytree(
-                src,
-                dst,
-                dirs_exist_ok=True,
-            )  # type: ignore
-        else:
-            shutil.copytree(src, dst)
-
     def _download_direct(
         self, checkpoint_storage: Dict[str, Any], local_ckpt_dir: pathlib.Path
     ) -> None:
         if checkpoint_storage["type"] == "shared_fs":
             src_ckpt_dir = self._find_shared_fs_path(checkpoint_storage)
-            self._shutil_copytree(str(src_ckpt_dir), str(local_ckpt_dir))
+            shutil.copytree(str(src_ckpt_dir), str(local_ckpt_dir), dirs_exist_ok=True)
         elif checkpoint_storage["type"] == "directory":
             src_ckpt_dir = pathlib.Path(checkpoint_storage["container_path"], self.uuid)
             if not src_ckpt_dir.exists():
@@ -301,7 +289,7 @@ def _download_direct(
                     "the same checkpoint storage directory present on the local machine as the "
                     "task runtime storage configuration.".format(self.uuid, src_ckpt_dir)
                 )
-            self._shutil_copytree(str(src_ckpt_dir), str(local_ckpt_dir))
+            shutil.copytree(str(src_ckpt_dir), str(local_ckpt_dir), dirs_exist_ok=True)
         else:
             local_ckpt_dir.mkdir(parents=True, exist_ok=True)
             manager = storage.build(

diff --git a/harness/mypy.ini b/harness/mypy.ini
@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.7
+python_version = 3.8
 follow_imports = silent
 ignore_missing_imports = True
 

diff --git a/model_hub/mypy.ini b/model_hub/mypy.ini
@@ -1,6 +1,6 @@
 [mypy]
 mypy_path = ../harness
-python_version = 3.7
+python_version = 3.8
 follow_imports = silent
 ignore_missing_imports = True
 

diff --git a/schemas/mypy.ini b/schemas/mypy.ini
@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.7
+python_version = 3.8
 follow_imports = silent
 ignore_missing_imports = True
 

diff --git a/tools/mypy.ini b/tools/mypy.ini
@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.7
+python_version = 3.8
 follow_imports = silent
 ignore_missing_imports = True
 

diff --git a/tools/scripts/bumpenvs.yaml b/tools/scripts/bumpenvs.yaml
@@ -86,22 +86,6 @@ pytorch20_tf210_rocm56_0_hashed: {new: determinedai/environments:rocm-5.6-pytorc
 pytorch20_tf210_rocm56_0_versioned: {new: determinedai/environments:rocm-5.6-pytorch-2.0-tf-2.10-rocm-ompi-0.27.1}
 pytorch20_tf210_rocm56_1_hashed: {new: determinedai/environments:rocm-5.6-pytorch-2.0-tf-2.10-rocm-ompi-f66cbce}
 pytorch20_tf210_rocm56_1_versioned: {new: determinedai/environments:rocm-5.6-pytorch-2.0-tf-2.10-rocm-ompi-0.27.1}
-tf1_cpu_0_hashed: {new: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-6eceaca,
-  old: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-a8b9c02}
-tf1_cpu_0_versioned: {new: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-0.21.2,
-  old: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-0.21.1}
-tf1_cpu_1_hashed: {new: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-mpi-6eceaca,
-  old: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-mpi-a8b9c02}
-tf1_cpu_1_versioned: {new: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-mpi-0.21.2,
-  old: determinedai/environments:py-3.7-pytorch-1.7-tf-1.15-cpu-mpi-0.21.1}
-tf1_gpu_0_hashed: {new: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-6eceaca,
-  old: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-a8b9c02}
-tf1_gpu_0_versioned: {new: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-0.21.2,
-  old: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-0.21.1}
-tf1_gpu_1_hashed: {new: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-mpi-6eceaca,
-  old: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-mpi-a8b9c02}
-tf1_gpu_1_versioned: {new: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-mpi-0.21.2,
-  old: determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-mpi-0.21.1}
 tf24_cpu_0_hashed: {new: determinedai/environments:py-3.8-pytorch-1.9-tf-2.4-cpu-24586f0,
   old: determinedai/environments-dev:py-3.8-pytorch-1.9-tf-2.4-cpu-1c769fb}
 tf24_cpu_0_versioned: {new: determinedai/environments:py-3.8-pytorch-1.9-tf-2.4-cpu-0.19.10,