chore: use tf 2.8 in ci [MLG-11] (#5711)

pin black to 21.7b0
type fixes for newer numpy that gets installed with tf 2.8

.circleci/config.yml

@@ -967,7 +967,7 @@ jobs:
       - setup-python-venv:
           install-python: false
           determined: true
-          extras-requires: "tensorflow==2.4.3 torch==1.9.0 torchvision==0.10.0"
+          extras-requires: "tensorflow==2.8.4 torch==1.9.0 torchvision==0.10.0"
           model-hub: true
           extra-requirements-file: "docs/requirements.txt"
           executor: <<pipeline.parameters.docker-image>>
@@ -1470,7 +1470,7 @@ jobs:
       - setup-python-venv:
           install-python: false
           determined: true
-          extras-requires: "tensorflow==2.4.3 torch==1.9.0 torchvision==0.10.0 codecov"
+          extras-requires: "tensorflow==2.8.4 torch==1.9.0 torchvision==0.10.0 codecov"
           extra-requirements-file: "harness/tests/requirements/requirements-harness.txt"
           executor: <<pipeline.parameters.docker-image>>
       - run: COVERAGE_FILE=$PWD/test-unit-harness-cpu-pycov make -C harness test-cpu
@@ -1491,7 +1491,7 @@ jobs:
       - skip-if-webui-only
       - setup-python-venv:
           determined: true
-          extras-requires: "tensorflow==2.4.3 torch==1.9.0 torchvision==0.10.0 codecov"
+          extras-requires: "tensorflow==2.8.4 torch==1.9.0 torchvision==0.10.0 codecov"
           extra-requirements-file: "harness/tests/requirements/requirements-harness.txt"
           executor: <<pipeline.parameters.docker-image>>
       - run: COVERAGE_FILE=$PWD/test-unit-harness-gpu-pycov make -C harness test-gpu
@@ -1512,7 +1512,7 @@ jobs:
       - setup-python-venv:
           install-python: false
           determined: true
-          extras-requires: "tensorflow==2.4.3 torch==1.9.0 codecov"
+          extras-requires: "tensorflow==2.8.4 torch==1.9.0 codecov"
           extra-requirements-file: "harness/tests/requirements/requirements-harness.txt"
           executor: <<pipeline.parameters.docker-image>>
       - run: COVERAGE_FILE=$PWD/test-unit-harness-tf2-pycov make -C harness test-tf2
@@ -1533,7 +1533,7 @@ jobs:
       - setup-python-venv:
           install-python: false
           determined: true
-          extras-requires: "tensorflow==2.4.3 torch==1.9.0 codecov"
+          extras-requires: "tensorflow==2.8.4 torch==1.9.0 codecov"
           extra-requirements-file: "harness/tests/requirements/requirements-harness.txt"
           executor: <<pipeline.parameters.docker-image>>
       - run: COVERAGE_FILE=$PWD/test-unit-storage-pycov coverage run -m pytest -v --durations=0 --require-secrets -m cloud harness/tests

.pre-commit-config.yaml

@@ -51,7 +51,7 @@ repos:
       - id: check-github-workflows
 
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.7b0
     hooks:
       - id: black
         # known issue fixed in a newer version:

CONTRIBUTING.md

@@ -133,17 +133,6 @@ In the future, ensure that you activate the virtualenv (by running the
 such as [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/)
 or [direnv](https://direnv.net/) may help streamline the process.
 
-### Building Errata
-
-If you are building Determined using Python 3.9, you must manually change the
-version of TensorFlow to 2.8.0 in the [end-to-end testing requirements file](https://github.com/determined-ai/determined/blob/master/e2e_tests/tests/requirements.txt#L8)
-because the version of TensorFlow currently set in the end-to-end requirements
-file is not compatible with Python 3.9:
-
-```plain
-tensorflow==2.8.0; sys_platform != 'darwin' or platform_machine != 'arm64'
-```
-
 ## Running Determined
 
 A minimal Determined cluster consists of three services: a

e2e_tests/tests/requirements.txt

@@ -5,8 +5,8 @@ pytest-timeout
 pexpect
 torch==1.9.0
 torchvision==0.10.0
-tensorflow==2.4.3; sys_platform != 'darwin' or platform_machine != 'arm64'
-tensorflow-macos==2.7.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow==2.8.4; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.8.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 pandas
 pyyaml
 docker

examples/tests/requirements.txt

@@ -1,7 +1,7 @@
 # pytest 6.0 has linter-breaking changes
 pytest>=6.0.1
-tensorflow==2.4.3; sys_platform != 'darwin' or platform_machine != 'arm64'
-tensorflow-macos==2.7.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow==2.8.4; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.8.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 torch==1.7.1
 torchvision==0.8.2
 pandas==1.0.3

harness/determined/pytorch/_reducer.py

@@ -35,13 +35,13 @@ def _simple_reduce_metrics(
                     "Lengths of metrics and num_batches are not equal: "
                     f"{len(metrics)} != {len(num_batches)}."
                 )
-        return np.average(metrics, weights=num_batches)
+        return np.average(metrics, weights=num_batches)  # type: ignore
     elif reducer == Reducer.SUM:
-        return np.sum(metrics)
+        return np.sum(metrics)  # type: ignore
     elif reducer == Reducer.MAX:
-        return np.max(metrics)
+        return np.max(metrics)  # type: ignore
     elif reducer == Reducer.MIN:
-        return np.min(metrics)
+        return np.min(metrics)  # type: ignore
     else:
         raise NotImplementedError
 

harness/determined/util.py

@@ -125,8 +125,8 @@ def make_metrics(num_inputs: Optional[int], batch_metrics: List[Dict[str, Any]])
     for name, values in metric_dict.items():
         m = None  # type: Optional[float]
         try:
-            values = np.array(values)
-            filtered_values = values[values != None]  # noqa: E711
+            np_values = np.array(values)
+            filtered_values = np_values[np_values != None]  # noqa: E711
             m = np.mean(filtered_values).item()
         except (TypeError, ValueError):
             # If we get here, values are non-scalars, which cannot be averaged.

harness/tests/experiment/fixtures/tf_keras_one_var_model.py

@@ -75,10 +75,12 @@ def keras_callbacks(self):
         epochs = self.context.get_hparams().get("epochs")
         validations = self.context.get_hparams().get("validations")
         # Include a bunch of callbacks just to make sure they work.
+        # EarlyStopping changed in TF 2.5 to stop unconditionally
+        # if patience=0
         return [
             keras_cb_checker.CBChecker(epochs=epochs, validations=validations),
             keras.callbacks.TensorBoard(),
             keras.callbacks.ReduceLROnPlateau(monitor="val_loss"),
-            keras.callbacks.EarlyStopping(restore_best_weights=True),
+            keras.callbacks.EarlyStopping(restore_best_weights=True, patience=1),
         ]
         return

harness/tests/experiment/pytorch/test_metric_utils.py

@@ -59,12 +59,12 @@ def test_process_combined_metrics_and_batches() -> None:
 
 
 def test_average_training_metrics() -> None:
-    combined_timeseries = {"loss1": [[1, 2], [3, 4]], "loss2": [[-1, -2], [-3, -4]]}
+    combined_timeseries: Dict[str, Any] = {"loss1": [[1, 2], [3, 4]], "loss2": [[-1, -2], [-3, -4]]}
     combined_num_batches = [2, 2]
     averaged_metrics = metric_utils._average_training_metrics(
         combined_timeseries, combined_num_batches
     )
-    expected_metrics = [
+    expected_metrics: List[Dict[str, Any]] = [
         {"loss1": 2, "loss2": -2},
         {"loss1": 3, "loss2": -3},
     ]

model_hub/model_hub/utils.py

@@ -28,7 +28,7 @@ def expand_like(arrays: List[np.ndarray], fill: float = -100) -> np.ndarray:
     """
     full_shape = list(arrays[0].shape)
     if len(full_shape) == 1:
-        return np.concatenate(arrays)
+        return np.concatenate(arrays)  # type: ignore
     full_shape[0] = sum(a.shape[0] for a in arrays)
     full_shape[1] = max(a.shape[1] for a in arrays)
     result = np.full(full_shape, fill)
@@ -48,7 +48,7 @@ def numpify(x: Union[List, np.ndarray, torch.Tensor]) -> np.ndarray:
     if isinstance(x, List):
         return np.array(x)
     if isinstance(x, torch.Tensor):
-        return x.cpu().numpy()
+        return x.cpu().numpy()  # type: ignore
     raise TypeError("Expected input of type List, np.ndarray, or torch.Tensor.")
 
 

requirements.txt

@@ -8,7 +8,7 @@
 -r model_hub/tests/requirements.txt
 -r e2e_tests/tests/requirements.txt
 
-black>=20.8b1
+black==21.7b0
 click<=8.0.4  # Unpin click after tensorflow update.
 flake8>=3.8.0
 flake8-bugbear>=19.8.0