From db3bf21164483cddcc4bac439813985b5375588f Mon Sep 17 00:00:00 2001 From: Alfonso Taboada Date: Wed, 20 Apr 2022 22:17:10 +0200 Subject: [PATCH 01/10] Fixed encoding issues on terminals that do not support unicode characters --- pytorch_lightning/loops/dataloader/evaluation_loop.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index 0dea13365a13f..1bd755f6066be 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import sys import shutil from collections import ChainMap, OrderedDict from functools import partial @@ -384,7 +385,15 @@ def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] row_format = f"{{:^{max_length}}}" * len(table_headers) half_term_size = int(term_size / 2) - bar = "─" * term_size + bar_character = "-" + try: + unicode_dash = "─" + unicode_dash.encode(sys.stdout.encoding) + bar_character = unicode_dash + except UnicodeEncodeError: + pass + bar = bar_character * term_size + lines = [bar, row_format.format(*table_headers).rstrip(), bar] for metric, row in zip(metrics, table_rows): # deal with column overflow From 8cf0c30bac5099f82b589cb759a0403567003024 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 20:25:24 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/loops/dataloader/evaluation_loop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index 1bd755f6066be..e72bb5a6c0d47 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import shutil +import sys from collections import ChainMap, OrderedDict from functools import partial from typing import Any, IO, Iterable, List, Optional, Sequence, Type, Union @@ -393,7 +393,7 @@ def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] except UnicodeEncodeError: pass bar = bar_character * term_size - + lines = [bar, row_format.format(*table_headers).rstrip(), bar] for metric, row in zip(metrics, table_rows): # deal with column overflow From 994ff5b4795690c0c4d309a4aa1d18ca7b45edff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 21 Apr 2022 15:37:36 +0200 Subject: [PATCH 03/10] Apply suggestions from code review --- pytorch_lightning/loops/dataloader/evaluation_loop.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index e72bb5a6c0d47..7abc4c40dfef4 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -13,7 +13,6 @@ # limitations under the License. import os import shutil -import sys from collections import ChainMap, OrderedDict from functools import partial from typing import Any, IO, Iterable, List, Optional, Sequence, Type, Union @@ -385,13 +384,13 @@ def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] row_format = f"{{:^{max_length}}}" * len(table_headers) half_term_size = int(term_size / 2) - bar_character = "-" try: - unicode_dash = "─" - unicode_dash.encode(sys.stdout.encoding) - bar_character = unicode_dash + # some terminals do not support this character + "─".encode(file.encoding) except UnicodeEncodeError: - pass + bar_character = "-" + else: + bar_character = "─" bar = bar_character * term_size lines = [bar, row_format.format(*table_headers).rstrip(), bar] From 3ab50b8a53d6f4b23d5aeea9f67f8747b53c9a70 Mon Sep 17 00:00:00 2001 From: Alfonso Taboada Date: Thu, 21 Apr 2022 22:59:53 +0200 Subject: [PATCH 04/10] Added test and fixed issue where file might be none or file.encoding might be absent --- CHANGELOG.md | 2 +- .../loops/dataloader/evaluation_loop.py | 15 +++++++---- .../logging_/test_eval_loop_logging.py | 26 +++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9398530102de7..168d070851d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,7 +129,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - When using custom DataLoaders in LightningDataModule, multiple inheritance is resolved properly ([#12716](https://github.com/PyTorchLightning/pytorch-lightning/pull/12716)) -- +- Fixed encoding issues on terminals that do not support unicode characters ## [1.6.1] - 2022-04-13 diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index 7abc4c40dfef4..566b3be86b19d 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import sys import shutil -from collections import ChainMap, OrderedDict -from functools import partial -from typing import Any, IO, Iterable, List, Optional, Sequence, Type, Union - import torch +from collections import ChainMap, OrderedDict from deprecate.utils import void +from functools import partial from torch.utils.data.dataloader import DataLoader +from typing import Any, IO, Iterable, List, Optional, Sequence, Type, Union import pytorch_lightning as pl from pytorch_lightning.accelerators import GPUAccelerator @@ -336,6 +336,10 @@ def _find_value(data: dict, target: str) -> Iterable[Any]: @staticmethod def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] = None) -> None: + # print to stdout by default + if file is None: + file = sys.stdout + # remove the dl idx suffix results = [{k.split("/dataloader_idx_")[0]: v for k, v in result.items()} for result in results] metrics = sorted({k for keys in apply_to_collection(results, dict, EvaluationLoop._get_keys) for k in keys}) @@ -386,7 +390,8 @@ def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] try: # some terminals do not support this character - "─".encode(file.encoding) + if hasattr(file, 'encoding') and file.encoding is not None: + "─".encode(file.encoding) except UnicodeEncodeError: bar_character = "-" else: diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index 8e2132498fd0e..8b600e548ae9d 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -870,6 +870,32 @@ def test_native_print_results(monkeypatch, inputs, expected): assert out.getvalue().replace(os.linesep, "\n") == expected.lstrip() + +inputs0 = ([{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}], RunningStage.TESTING) + +@pytest.mark.parametrize( + ["inputs", "encoding"], + [ + pytest.param(inputs0, 'latin-1', id="case0"), + pytest.param(inputs0, 'utf-8', id="case1") + ], +) +def test_native_print_results_encodings(monkeypatch, inputs, encoding): + import pytorch_lightning.loops.dataloader.evaluation_loop as imports + + monkeypatch.setattr(imports, "_RICH_AVAILABLE", False) + out = mock.Mock() + out.encoding = encoding + EvaluationLoop._print_results(*inputs, file=out) + + # Attempt to encode everything the file is told to write with the given encoding + for call in out.method_calls: + name, args, kwargs = call + if name == 'write': + args[0].encode(encoding) + + + expected0 = """ ┏━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ Test metric ┃ DataLoader 0 ┃ DataLoader 1 ┃ From ed2992662a082462d7907d25e4b34053e79cd4b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:03:51 +0000 Subject: [PATCH 05/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/loops/dataloader/evaluation_loop.py | 11 ++++++----- tests/trainer/logging_/test_eval_loop_logging.py | 10 +++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index 566b3be86b19d..b8c1cc9550475 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys import shutil -import torch +import sys from collections import ChainMap, OrderedDict -from deprecate.utils import void from functools import partial -from torch.utils.data.dataloader import DataLoader from typing import Any, IO, Iterable, List, Optional, Sequence, Type, Union +import torch +from deprecate.utils import void +from torch.utils.data.dataloader import DataLoader + import pytorch_lightning as pl from pytorch_lightning.accelerators import GPUAccelerator from pytorch_lightning.loops.dataloader import DataLoaderLoop @@ -390,7 +391,7 @@ def _print_results(results: List[_OUT_DICT], stage: str, file: Optional[IO[str]] try: # some terminals do not support this character - if hasattr(file, 'encoding') and file.encoding is not None: + if hasattr(file, "encoding") and file.encoding is not None: "─".encode(file.encoding) except UnicodeEncodeError: bar_character = "-" diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index 8b600e548ae9d..b752b8bc5b926 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -870,15 +870,12 @@ def test_native_print_results(monkeypatch, inputs, expected): assert out.getvalue().replace(os.linesep, "\n") == expected.lstrip() - inputs0 = ([{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}], RunningStage.TESTING) + @pytest.mark.parametrize( ["inputs", "encoding"], - [ - pytest.param(inputs0, 'latin-1', id="case0"), - pytest.param(inputs0, 'utf-8', id="case1") - ], + [pytest.param(inputs0, "latin-1", id="case0"), pytest.param(inputs0, "utf-8", id="case1")], ) def test_native_print_results_encodings(monkeypatch, inputs, encoding): import pytorch_lightning.loops.dataloader.evaluation_loop as imports @@ -891,11 +888,10 @@ def test_native_print_results_encodings(monkeypatch, inputs, encoding): # Attempt to encode everything the file is told to write with the given encoding for call in out.method_calls: name, args, kwargs = call - if name == 'write': + if name == "write": args[0].encode(encoding) - expected0 = """ ┏━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ Test metric ┃ DataLoader 0 ┃ DataLoader 1 ┃ From d9bc76e3d48d2a660bfa4bbc7f182c9f99c25dc6 Mon Sep 17 00:00:00 2001 From: Alfonso Taboada Date: Thu, 21 Apr 2022 23:13:00 +0200 Subject: [PATCH 06/10] Fix PEP8 --- tests/trainer/logging_/test_eval_loop_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index b752b8bc5b926..d546e5fc7ea0e 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -886,8 +886,8 @@ def test_native_print_results_encodings(monkeypatch, inputs, encoding): EvaluationLoop._print_results(*inputs, file=out) # Attempt to encode everything the file is told to write with the given encoding - for call in out.method_calls: - name, args, kwargs = call + for call_ in out.method_calls: + name, args, kwargs = call_ if name == "write": args[0].encode(encoding) From 6c9201bc7d18925d9065db3da31c7c9af8370d05 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Sun, 24 Apr 2022 03:01:10 +0900 Subject: [PATCH 07/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9e2de4ed97a..ea9159b7046ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,7 +135,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - When using custom DataLoaders in LightningDataModule, multiple inheritance is resolved properly ([#12716](https://github.com/PyTorchLightning/pytorch-lightning/pull/12716)) -- Fixed encoding issues on terminals that do not support unicode characters ([#12823](https://github.com/PyTorchLightning/pytorch-lightning/issues/12823)) +- Fixed encoding issues on terminals that do not support unicode characters ([#12828](https://github.com/PyTorchLightning/pytorch-lightning/pull/12828)) - Fixed support for `ModelCheckpoint` monitors with dots ([#12783](https://github.com/PyTorchLightning/pytorch-lightning/pull/12783)) From 9eb47b514b3fb7cb5f5d3677e315de558b91e835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Sat, 23 Apr 2022 20:15:24 +0200 Subject: [PATCH 08/10] Update tests/trainer/logging_/test_eval_loop_logging.py Co-authored-by: Akihiro Nitta --- tests/trainer/logging_/test_eval_loop_logging.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index d546e5fc7ea0e..31c0c7ef42c17 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -870,20 +870,16 @@ def test_native_print_results(monkeypatch, inputs, expected): assert out.getvalue().replace(os.linesep, "\n") == expected.lstrip() -inputs0 = ([{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}], RunningStage.TESTING) - - -@pytest.mark.parametrize( - ["inputs", "encoding"], - [pytest.param(inputs0, "latin-1", id="case0"), pytest.param(inputs0, "utf-8", id="case1")], -) -def test_native_print_results_encodings(monkeypatch, inputs, encoding): +@pytest.mark.parametrize("encoding", ["latin-1", "utf-8"]) +def test_native_print_results_encodings(monkeypatch, encoding): import pytorch_lightning.loops.dataloader.evaluation_loop as imports monkeypatch.setattr(imports, "_RICH_AVAILABLE", False) + + results = [{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}] out = mock.Mock() out.encoding = encoding - EvaluationLoop._print_results(*inputs, file=out) + EvaluationLoop._print_results(results, RunningStage.TESTING, file=out) # Attempt to encode everything the file is told to write with the given encoding for call_ in out.method_calls: From 0da4d44dc0c9662e3584f93477a67c18bb839404 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Apr 2022 18:16:36 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/trainer/logging_/test_eval_loop_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index 31c0c7ef42c17..fad51294b7c6a 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -875,7 +875,7 @@ def test_native_print_results_encodings(monkeypatch, encoding): import pytorch_lightning.loops.dataloader.evaluation_loop as imports monkeypatch.setattr(imports, "_RICH_AVAILABLE", False) - + results = [{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}] out = mock.Mock() out.encoding = encoding From a9a0cb3a2cf33970bd0fd654f78f2d14698a1c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Sat, 23 Apr 2022 20:23:44 +0200 Subject: [PATCH 10/10] Update tests/trainer/logging_/test_eval_loop_logging.py --- tests/trainer/logging_/test_eval_loop_logging.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/trainer/logging_/test_eval_loop_logging.py b/tests/trainer/logging_/test_eval_loop_logging.py index fad51294b7c6a..0ca9bf3107b9c 100644 --- a/tests/trainer/logging_/test_eval_loop_logging.py +++ b/tests/trainer/logging_/test_eval_loop_logging.py @@ -876,10 +876,9 @@ def test_native_print_results_encodings(monkeypatch, encoding): monkeypatch.setattr(imports, "_RICH_AVAILABLE", False) - results = [{"log": torch.tensor(5)}, {"no_log": torch.tensor(6)}] out = mock.Mock() out.encoding = encoding - EvaluationLoop._print_results(results, RunningStage.TESTING, file=out) + EvaluationLoop._print_results(*inputs0, file=out) # Attempt to encode everything the file is told to write with the given encoding for call_ in out.method_calls: