From f56ca3353c0fc076f4395e7448f49a948fb83569 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Sun, 8 Aug 2021 04:37:28 +0700 Subject: [PATCH 01/18] LGBM integration added --- dvclive/lgbm.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 dvclive/lgbm.py diff --git a/dvclive/lgbm.py b/dvclive/lgbm.py new file mode 100644 index 00000000..3a0b734f --- /dev/null +++ b/dvclive/lgbm.py @@ -0,0 +1,17 @@ +import dvclive + + +class DvcLiveCallback: + def __init__(self, model_file=None): + super().__init__() + self.model_file = model_file + + def __call__(self, env): + for eval_result in env.evaluation_result_list: + metric = eval_result[1] + value = eval_result[2] + dvclive.log(metric, value) + dvclive.next_step() + + if self.model_file and env.iteration == env.end_iteration - 1: + env.model.save_model(self.model_file) From ccae1aa7a3e0ba3cce3440bd9060529fb9821bec Mon Sep 17 00:00:00 2001 From: pacifikus Date: Sun, 8 Aug 2021 21:53:49 +0700 Subject: [PATCH 02/18] Tests for lgbm integration --- setup.py | 4 ++- tests/test_lgbm.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/test_lgbm.py diff --git a/setup.py b/setup.py index 7cfd159b..56a0a8a0 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,9 @@ def run(self): mmcv = ["mmcv", "torch", "torchvision"] tf = ["tensorflow"] xgb = ["xgboost"] +lgbm = ["lightgbm"] -all_libs = mmcv + tf + xgb +all_libs = mmcv + tf + xgb + lgbm tests_requires = [ "pylint==2.5.3", @@ -71,6 +72,7 @@ def run(self): "all": all_libs, "tf": tf, "xgb": xgb, + "lgbm": lgbm, }, keywords="data-science metrics machine-learning developer-tools ai", python_requires=">=3.6", diff --git a/tests/test_lgbm.py b/tests/test_lgbm.py new file mode 100644 index 00000000..1c71c83c --- /dev/null +++ b/tests/test_lgbm.py @@ -0,0 +1,71 @@ +import os + +import lightgbm as lgbm +import numpy as np +import pandas as pd +import pytest +from funcy import first +from sklearn import datasets +from sklearn.model_selection import train_test_split + +import dvclive +from dvclive.lgbm import DvcLiveCallback +from tests.test_main import read_logs + +# pylint: disable=redefined-outer-name, unused-argument + + +@pytest.fixture +def model_params(): + return {"objective": "multiclass", "n_estimators": 5, "seed": 0} + + +@pytest.fixture +def iris_data(): + iris = datasets.load_iris() + x = pd.DataFrame(iris["data"], columns=iris["feature_names"]) + y = iris["target"] + x_train, x_test, y_train, y_test = train_test_split( + x, y, test_size=0.33, random_state=42 + ) + return (x_train, y_train), (x_test, y_test) + + +def test_lgbm_integration(model_params, iris_data): + dvclive.init("logs") + model = lgbm.LGBMClassifier() + model.set_params(**model_params) + + model.fit( + iris_data[0][0], + iris_data[0][1], + eval_set=(iris_data[1][0], iris_data[1][1]), + eval_metric=["multi_logloss"], + callbacks=[DvcLiveCallback()], + ) + + assert os.path.exists("logs") + + logs, _ = read_logs("logs") + assert len(logs) == 1 + assert len(first(logs.values())) == 5 + + +def test_lgbm_model_file(model_params, iris_data): + dvclive.init("logs") + model = lgbm.LGBMClassifier() + model.set_params(**model_params) + + model.fit( + iris_data[0][0], + iris_data[0][1], + eval_set=(iris_data[1][0], iris_data[1][1]), + eval_metric=["multi_logloss"], + callbacks=[DvcLiveCallback("lgbm_model")], + ) + + preds = model.predict(iris_data[1][0]) + model2 = lgbm.Booster(model_file="lgbm_model") + preds2 = model2.predict(iris_data[1][0]) + preds2 = np.argmax(preds2, axis=1) + assert np.sum(np.abs(preds2 - preds)) == 0 From 954294948c6d6431a27e19e725a8ce6d662298b1 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Sun, 8 Aug 2021 22:03:26 +0700 Subject: [PATCH 03/18] test_lgbm: added tmp_dir fixture --- tests/test_lgbm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lgbm.py b/tests/test_lgbm.py index 1c71c83c..8195f5fc 100644 --- a/tests/test_lgbm.py +++ b/tests/test_lgbm.py @@ -31,7 +31,7 @@ def iris_data(): return (x_train, y_train), (x_test, y_test) -def test_lgbm_integration(model_params, iris_data): +def test_lgbm_integration(tmp_dir, model_params, iris_data): dvclive.init("logs") model = lgbm.LGBMClassifier() model.set_params(**model_params) @@ -51,7 +51,7 @@ def test_lgbm_integration(model_params, iris_data): assert len(first(logs.values())) == 5 -def test_lgbm_model_file(model_params, iris_data): +def test_lgbm_model_file(tmp_dir, model_params, iris_data): dvclive.init("logs") model = lgbm.LGBMClassifier() model.set_params(**model_params) From 71092cfa97ceb675f3ced58dee1b216ec1f8d011 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Mon, 9 Aug 2021 12:16:28 +0400 Subject: [PATCH 04/18] Update dvclive/lgbm.py Save model at the end of each iteration Co-authored-by: David de la Iglesia Castro --- dvclive/lgbm.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dvclive/lgbm.py b/dvclive/lgbm.py index 3a0b734f..a49b55c0 100644 --- a/dvclive/lgbm.py +++ b/dvclive/lgbm.py @@ -11,7 +11,6 @@ def __call__(self, env): metric = eval_result[1] value = eval_result[2] dvclive.log(metric, value) - dvclive.next_step() - - if self.model_file and env.iteration == env.end_iteration - 1: + if self.model_file: env.model.save_model(self.model_file) + dvclive.next_step() From 295c6b7cf01d5e90bb287cfe7bdb566fe200195c Mon Sep 17 00:00:00 2001 From: pacifikus Date: Fri, 13 Aug 2021 17:13:55 +0700 Subject: [PATCH 05/18] Huggingface integration --- dvclive/huggingface.py | 31 +++++++++++ setup.py | 4 +- tests/test_huggingface.py | 108 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 dvclive/huggingface.py create mode 100644 tests/test_huggingface.py diff --git a/dvclive/huggingface.py b/dvclive/huggingface.py new file mode 100644 index 00000000..a40dc76e --- /dev/null +++ b/dvclive/huggingface.py @@ -0,0 +1,31 @@ +from transformers import ( + TrainerCallback, + TrainerControl, + TrainerState, + TrainingArguments, +) + +import dvclive + + +class DvcLiveCallback(TrainerCallback): + def __init__(self, model_file=None): + super().__init__() + self.model_file = model_file + + def on_evaluate( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs + ): + metrics = kwargs["metrics"] + for key, value in metrics.items(): + dvclive.log(key, value) + + if self.model_file: + model = kwargs["model"] + model.save_pretrained(self.model_file) + + dvclive.next_step() diff --git a/setup.py b/setup.py index 56a0a8a0..c37b139e 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,9 @@ def run(self): tf = ["tensorflow"] xgb = ["xgboost"] lgbm = ["lightgbm"] +hugginface = ["transformers", "datasets"] -all_libs = mmcv + tf + xgb + lgbm +all_libs = mmcv + tf + xgb + lgbm + hugginface tests_requires = [ "pylint==2.5.3", @@ -73,6 +74,7 @@ def run(self): "tf": tf, "xgb": xgb, "lgbm": lgbm, + "huggingface": hugginface, }, keywords="data-science metrics machine-learning developer-tools ai", python_requires=">=3.6", diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py new file mode 100644 index 00000000..a1786222 --- /dev/null +++ b/tests/test_huggingface.py @@ -0,0 +1,108 @@ +import os + +import numpy as np +import pytest +from datasets import load_dataset, load_metric +from funcy import first +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + Trainer, + TrainingArguments, +) + +import dvclive +from dvclive.huggingface import DvcLiveCallback +from tests.test_main import read_logs + +# pylint: disable=redefined-outer-name, unused-argument, no-value-for-parameter + +task = "cola" +metric = load_metric("glue", task) +model_checkpoint = "distilbert-base-uncased" + + +def preprocess_function(examples, tokenizer): + return tokenizer(examples["sentence"], truncation=True) + + +def compute_metrics(eval_pred): + predictions, labels = eval_pred + predictions = np.argmax(predictions, axis=1) + return metric.compute(predictions=predictions, references=labels) + + +@pytest.fixture +def tokenizer(): + return AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True) + + +@pytest.fixture +def data(tokenizer): + train = load_dataset("glue", task, split="train[:100]") + val = load_dataset("glue", task, split="validation[:100]") + + train = train.map( + lambda p: preprocess_function(p, tokenizer), batched=True + ) + val = val.map(lambda p: preprocess_function(p, tokenizer), batched=True) + + return train, val + + +@pytest.fixture +def model(): + return AutoModelForSequenceClassification.from_pretrained( + pretrained_model_name_or_path=model_checkpoint, num_labels=2 + ) + + +@pytest.fixture +def args(): + return TrainingArguments( + "test-glue", evaluation_strategy="epoch", num_train_epochs=2, + ) + + +def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): + dvclive.init("logs") + + trainer = Trainer( + model, + args, + train_dataset=data[0], + eval_dataset=data[1], + tokenizer=tokenizer, + compute_metrics=compute_metrics, + ) + trainer.add_callback(DvcLiveCallback()) + trainer.train() + + assert os.path.exists("logs") + + logs, _ = read_logs("logs") + + assert len(logs) == 6 + assert "eval_matthews_correlation" in logs + assert "eval_loss" in logs + assert len(first(logs.values())) == 2 + + +def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): + dvclive.init("logs") + model_path = tmp_dir / "model_hf" + + trainer = Trainer( + model, + args, + train_dataset=data[0], + eval_dataset=data[1], + tokenizer=tokenizer, + compute_metrics=compute_metrics, + ) + trainer.add_callback(DvcLiveCallback(model_file=model_path)) + trainer.train() + + assert model_path.is_dir() + assert (model_path / "pytorch_model.bin").exists() + assert (model_path / "config.json").exists() From 92d8a8e5e307683169f12be9afccb395a19cec27 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Sun, 15 Aug 2021 00:58:00 +0700 Subject: [PATCH 06/18] Add on_log event --- dvclive/huggingface.py | 17 ++++++++++++++--- tests/test_huggingface.py | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dvclive/huggingface.py b/dvclive/huggingface.py index a40dc76e..4cc13b9c 100644 --- a/dvclive/huggingface.py +++ b/dvclive/huggingface.py @@ -24,8 +24,19 @@ def on_evaluate( for key, value in metrics.items(): dvclive.log(key, value) - if self.model_file: - model = kwargs["model"] - model.save_pretrained(self.model_file) + def on_log( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs + ): + logs = kwargs["logs"] + for key, value in logs.items(): + if "train_" in key: + dvclive.log(key, value) + if self.model_file: + model = kwargs["model"] + model.save_pretrained(self.model_file) dvclive.next_step() diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index a1786222..dc6dec22 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -82,7 +82,7 @@ def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): logs, _ = read_logs("logs") - assert len(logs) == 6 + assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs assert len(first(logs.values())) == 2 From 62dabc9c27d92a2eb8acf48220379f527535b563 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 17 Aug 2021 00:23:25 +0400 Subject: [PATCH 07/18] Update dvclive/huggingface.py Co-authored-by: David de la Iglesia Castro --- dvclive/huggingface.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/dvclive/huggingface.py b/dvclive/huggingface.py index 4cc13b9c..bbc02b50 100644 --- a/dvclive/huggingface.py +++ b/dvclive/huggingface.py @@ -13,17 +13,6 @@ def __init__(self, model_file=None): super().__init__() self.model_file = model_file - def on_evaluate( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - **kwargs - ): - metrics = kwargs["metrics"] - for key, value in metrics.items(): - dvclive.log(key, value) - def on_log( self, args: TrainingArguments, @@ -33,8 +22,7 @@ def on_log( ): logs = kwargs["logs"] for key, value in logs.items(): - if "train_" in key: - dvclive.log(key, value) + dvclive.log(key, value) if self.model_file: model = kwargs["model"] From 5895ee19ac995ed6f867a5dd1bcbc71d622918e3 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 17 Aug 2021 03:23:54 +0700 Subject: [PATCH 08/18] fix: huggingface test after callback changes --- tests/test_huggingface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index dc6dec22..c969539c 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -64,7 +64,7 @@ def args(): ) -def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): +def test_huggingface_integration(model, args, data, tokenizer): dvclive.init("logs") trainer = Trainer( @@ -85,7 +85,7 @@ def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs - assert len(first(logs.values())) == 2 + assert len(first(logs.values())) == 3 def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): From baaa7949a061995a0cb51bf89ce4d8eede3b757c Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 17 Aug 2021 03:43:26 +0700 Subject: [PATCH 09/18] revert last commit --- tests/test_huggingface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index c969539c..dc6dec22 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -64,7 +64,7 @@ def args(): ) -def test_huggingface_integration(model, args, data, tokenizer): +def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): dvclive.init("logs") trainer = Trainer( @@ -85,7 +85,7 @@ def test_huggingface_integration(model, args, data, tokenizer): assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs - assert len(first(logs.values())) == 3 + assert len(first(logs.values())) == 2 def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): From 438fe54f19f8516dc37916a123bb871ba9994b6a Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 17 Aug 2021 03:55:58 +0700 Subject: [PATCH 10/18] fix: huggingface test after calback changes --- tests/test_huggingface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index dc6dec22..c6bc8b64 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -85,7 +85,7 @@ def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs - assert len(first(logs.values())) == 2 + assert len(first(logs.values())) == 3 def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): From 94210a015cedc43d5fe88ca88a4a86d8acd50020 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 18 Aug 2021 17:20:09 +0200 Subject: [PATCH 11/18] Updated test_huggingface --- tests/test_huggingface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index c6bc8b64..5c084085 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -85,7 +85,8 @@ def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs - assert len(first(logs.values())) == 3 + assert len(logs["epoch"]) == 3 + assert len(logs["eval_loss"]) == 2 def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): From 9b484404c99dcff8a189624943de337257ed6fce Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 18 Aug 2021 17:24:21 +0200 Subject: [PATCH 12/18] Updated test_huggingface --- tests/test_huggingface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py index 5c084085..b1a8dd75 100644 --- a/tests/test_huggingface.py +++ b/tests/test_huggingface.py @@ -3,7 +3,6 @@ import numpy as np import pytest from datasets import load_dataset, load_metric -from funcy import first from transformers import ( AutoModelForSequenceClassification, AutoTokenizer, From 1c43c33e5956c93df7ffc0d1d8201f84069808e1 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 03:17:03 +0700 Subject: [PATCH 13/18] Catalyst integration --- dvclive/catalyst.py | 28 ++++++++++++ setup.py | 4 +- tests/test_catalyst.py | 101 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 dvclive/catalyst.py create mode 100644 tests/test_catalyst.py diff --git a/dvclive/catalyst.py b/dvclive/catalyst.py new file mode 100644 index 00000000..a5ed3b81 --- /dev/null +++ b/dvclive/catalyst.py @@ -0,0 +1,28 @@ +from catalyst.core.callback import Callback, CallbackOrder + +import dvclive + + +class DVCCallback(Callback): + def __init__(self, model_file=None): + super().__init__(order=CallbackOrder.external) + self.model_file = model_file + + def on_epoch_end(self, runner) -> None: + step = runner.stage_epoch_step + + for loader_key, per_loader_metrics in runner.epoch_metrics.items(): + for key, value in per_loader_metrics.items(): + key = key.replace("/", "_") + key += "_" + loader_key + dvclive.log(key, float(value), step) + + if self.model_file: + checkpoint = runner.engine.pack_checkpoint( + model=runner.model, + criterion=runner.criterion, + optimizer=runner.optimizer, + scheduler=runner.scheduler, + ) + runner.engine.save_checkpoint(checkpoint, self.model_file) + dvclive.next_step() diff --git a/setup.py b/setup.py index c37b139e..906e0898 100644 --- a/setup.py +++ b/setup.py @@ -41,8 +41,9 @@ def run(self): xgb = ["xgboost"] lgbm = ["lightgbm"] hugginface = ["transformers", "datasets"] +catalyst = ["catalyst"] -all_libs = mmcv + tf + xgb + lgbm + hugginface +all_libs = mmcv + tf + xgb + lgbm + hugginface + catalyst tests_requires = [ "pylint==2.5.3", @@ -75,6 +76,7 @@ def run(self): "xgb": xgb, "lgbm": lgbm, "huggingface": hugginface, + "catalyst": catalyst, }, keywords="data-science metrics machine-learning developer-tools ai", python_requires=">=3.6", diff --git a/tests/test_catalyst.py b/tests/test_catalyst.py new file mode 100644 index 00000000..fd330fe1 --- /dev/null +++ b/tests/test_catalyst.py @@ -0,0 +1,101 @@ +import os + +import pytest +from catalyst import dl +from catalyst.contrib.datasets import MNIST +from catalyst.data import ToTensor +from catalyst.utils.torch import get_available_engine +from torch import nn, optim +from torch.utils.data import DataLoader + +import dvclive +from dvclive.catalyst import DVCCallback +from tests.test_main import read_logs + +# pylint: disable=redefined-outer-name, unused-argument + + +@pytest.fixture +def loaders(): + train_data = MNIST( + os.getcwd(), train=True, download=True, transform=ToTensor() + ) + valid_data = MNIST( + os.getcwd(), train=False, download=True, transform=ToTensor() + ) + return { + "train": DataLoader(train_data, batch_size=32), + "valid": DataLoader(valid_data, batch_size=32), + } + + +@pytest.fixture +def runner(): + return dl.SupervisedRunner( + engine=get_available_engine(), + input_key="features", + output_key="logits", + target_key="targets", + loss_key="loss", + ) + + +def test_catalyst_callback(tmp_dir, runner, loaders): + dvclive.init("dvc_logs") + + model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10)) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.02) + + runner.train( + model=model, + criterion=criterion, + optimizer=optimizer, + loaders=loaders, + num_epochs=2, + callbacks=[ + dl.AccuracyCallback(input_key="logits", target_key="targets"), + DVCCallback(), + ], + logdir="./logs", + valid_loader="valid", + valid_metric="loss", + minimize_valid_metric=True, + verbose=True, + load_best_on_end=True, + ) + + assert os.path.exists("dvc_logs") + logs, _ = read_logs("dvc_logs") + + assert "accuracy_train" in logs + assert "loss_valid" in logs + assert len(logs["accuracy_train"]) == 2 + + +def test_catalyst_model_file(tmp_dir, runner, loaders): + dvclive.init("dvc_logs") + + model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10)) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.02) + + runner.train( + model=model, + engine=runner.engine, + criterion=criterion, + optimizer=optimizer, + loaders=loaders, + num_epochs=2, + callbacks=[ + dl.AccuracyCallback(input_key="logits", target_key="targets"), + DVCCallback("model.pth"), + ], + logdir="./logs", + valid_loader="valid", + valid_metric="loss", + minimize_valid_metric=True, + verbose=True, + load_best_on_end=True, + ) + assert (tmp_dir / "model.pth").is_file() From 6407fd12acebf571dd025d4897e3fa6b8fb23108 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 03:21:53 +0700 Subject: [PATCH 14/18] Callback rename --- dvclive/catalyst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvclive/catalyst.py b/dvclive/catalyst.py index a5ed3b81..ef51093b 100644 --- a/dvclive/catalyst.py +++ b/dvclive/catalyst.py @@ -3,7 +3,7 @@ import dvclive -class DVCCallback(Callback): +class DvcLiveCallback(Callback): def __init__(self, model_file=None): super().__init__(order=CallbackOrder.external) self.model_file = model_file From 1349cb0300334fc63eb7e6a585251a40be12afd2 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 03:23:35 +0700 Subject: [PATCH 15/18] Rename callback in test --- tests/test_catalyst.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_catalyst.py b/tests/test_catalyst.py index fd330fe1..756e4fd8 100644 --- a/tests/test_catalyst.py +++ b/tests/test_catalyst.py @@ -9,7 +9,7 @@ from torch.utils.data import DataLoader import dvclive -from dvclive.catalyst import DVCCallback +from dvclive.catalyst import DvcLiveCallback from tests.test_main import read_logs # pylint: disable=redefined-outer-name, unused-argument @@ -55,7 +55,7 @@ def test_catalyst_callback(tmp_dir, runner, loaders): num_epochs=2, callbacks=[ dl.AccuracyCallback(input_key="logits", target_key="targets"), - DVCCallback(), + DvcLiveCallback(), ], logdir="./logs", valid_loader="valid", @@ -89,7 +89,7 @@ def test_catalyst_model_file(tmp_dir, runner, loaders): num_epochs=2, callbacks=[ dl.AccuracyCallback(input_key="logits", target_key="targets"), - DVCCallback("model.pth"), + DvcLiveCallback("model.pth"), ], logdir="./logs", valid_loader="valid", From 70f68673cc816394a492c4c01faf467ded6fb257 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 12:51:51 +0400 Subject: [PATCH 16/18] Update dvclive/catalyst.py Co-authored-by: David de la Iglesia Castro --- dvclive/catalyst.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dvclive/catalyst.py b/dvclive/catalyst.py index ef51093b..bfd3fe71 100644 --- a/dvclive/catalyst.py +++ b/dvclive/catalyst.py @@ -14,8 +14,7 @@ def on_epoch_end(self, runner) -> None: for loader_key, per_loader_metrics in runner.epoch_metrics.items(): for key, value in per_loader_metrics.items(): key = key.replace("/", "_") - key += "_" + loader_key - dvclive.log(key, float(value), step) + dvclive.log(f"{loader_key}/{key}", float(value), step) if self.model_file: checkpoint = runner.engine.pack_checkpoint( From 59c88fa7e2101255d4155b9a46d6583d11790b45 Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 16:24:19 +0700 Subject: [PATCH 17/18] upd: catalyst tests --- tests/test_catalyst.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_catalyst.py b/tests/test_catalyst.py index 756e4fd8..3bb46180 100644 --- a/tests/test_catalyst.py +++ b/tests/test_catalyst.py @@ -68,9 +68,9 @@ def test_catalyst_callback(tmp_dir, runner, loaders): assert os.path.exists("dvc_logs") logs, _ = read_logs("dvc_logs") - assert "accuracy_train" in logs - assert "loss_valid" in logs - assert len(logs["accuracy_train"]) == 2 + assert "train\\accuracy" in logs + assert "valid\\loss" in logs + assert len(logs["train\\accuracy"]) == 2 def test_catalyst_model_file(tmp_dir, runner, loaders): From 38360f3f1fe12b44602b71682b1e6d80598b681d Mon Sep 17 00:00:00 2001 From: pacifikus Date: Tue, 24 Aug 2021 16:48:41 +0700 Subject: [PATCH 18/18] Fix: cross platform tests --- tests/test_catalyst.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_catalyst.py b/tests/test_catalyst.py index 3bb46180..f759d26f 100644 --- a/tests/test_catalyst.py +++ b/tests/test_catalyst.py @@ -10,7 +10,6 @@ import dvclive from dvclive.catalyst import DvcLiveCallback -from tests.test_main import read_logs # pylint: disable=redefined-outer-name, unused-argument @@ -66,11 +65,13 @@ def test_catalyst_callback(tmp_dir, runner, loaders): ) assert os.path.exists("dvc_logs") - logs, _ = read_logs("dvc_logs") - assert "train\\accuracy" in logs - assert "valid\\loss" in logs - assert len(logs["train\\accuracy"]) == 2 + train_path = tmp_dir / "dvc_logs/train" + valid_path = tmp_dir / "dvc_logs/valid" + + assert train_path.is_dir() + assert valid_path.is_dir() + assert (train_path / "accuracy.tsv").exists() def test_catalyst_model_file(tmp_dir, runner, loaders):