[#22] fixed the logic for computing bleu scores. black-formatted all …

…files

cleanformer/fetchers.py

@@ -34,7 +34,7 @@ def fetch_tokenizer(name: str) -> Tokenizer:
     tokenizer.bos_token_id = artifact.metadata["bos_id"]
     tokenizer.eos_token = artifact.metadata["eos"]
     tokenizer.eos_token_id = artifact.metadata["eos_id"]
-    tokenizer.kor2eng = artifact.metadata['kor2eng']
+    tokenizer.kor2eng = artifact.metadata["kor2eng"]
     return tokenizer
 
 

cleanformer/logcallback.py

@@ -11,6 +11,7 @@ class LogCallback(Callback):
     """
     For logging loss, perplexity, accuracy, BLEU along with qualitative results.
     """
+
     def __init__(self, tokenizer: Tokenizer):
         self.tokenizer = tokenizer
         self.cache = {"train": dict(), "validation": dict(), "test": dict()}
@@ -26,15 +27,25 @@ def on_validation_epoch_start(self, *args, **kwargs) -> None:
     def on_test_epoch_start(self, *args, **kwargs) -> None:
         self.cache["test"].clear()
 
-    def on_any_batch_end(self, key: str, transformer: Transformer,
-                         src: torch.Tensor, tgt_r: torch.Tensor, tgt_ids: torch.Tensor, losses: List[float]) -> tuple:
+    def on_any_batch_end(
+        self,
+        key: str,
+        transformer: Transformer,
+        src: torch.Tensor,
+        tgt_r: torch.Tensor,
+        tgt_ids: torch.Tensor,
+        losses: List[float],
+    ) -> Tuple[List[List[List[str]]], List[List[str]]]:
         inputs = self.tokenizer.decode_batch(src[:, 0].cpu().tolist())
         answers = self.tokenizer.decode_batch(tgt_ids.cpu().tolist())
         predictions = self.tokenizer.decode_batch(transformer.infer(src, tgt_r).cpu().tolist())
         self.cache[key]["inputs"] = self.cache[key].get("inputs", list()) + inputs
         self.cache[key]["answers"] = self.cache[key].get("answers", list()) + answers
         self.cache[key]["predictions"] = self.cache[key].get("predictions", list()) + predictions
         self.cache[key]["losses"] = self.cache[key].get("losses", list()) + losses
+        # to make them compatible with torchmetrics.functional.bleu_score()
+        answers = [[answer.split()] for answer in answers]
+        predictions = [prediction.split() for prediction in predictions]
         return answers, predictions
 
     @torch.no_grad()
@@ -45,15 +56,28 @@ def on_train_batch_end(
         out: dict,
         batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
         *args,
-        **kwargs
+        **kwargs,
     ) -> None:
         src, tgt_r, tgt_ids = batch
         transformer.log("train/loss", out["loss"], on_step=True, on_epoch=True)
         transformer.log("train/perplexity", torch.exp(out["loss"]), on_step=True, on_epoch=True)
-        transformer.log("train/accuracy", metricsF.accuracy(out["logits"], tgt_ids), on_step=True, on_epoch=True)
-        answers, predictions = self.on_any_batch_end("train", transformer, src, tgt_r, tgt_ids,
-                                                     out['losses'].cpu().tolist())
-        transformer.log("train/bleu", metricsF.bleu_score(answers, predictions), on_step=True, on_epoch=True)
+        transformer.log(
+            "train/accuracy", metricsF.accuracy(out["logits"], tgt_ids), on_step=True, on_epoch=True
+        )
+        answers, predictions = self.on_any_batch_end(
+            "train", transformer, src, tgt_r, tgt_ids, out["losses"].cpu().tolist()
+        )
+        transformer.log(
+            "train/bleu",
+            metricsF.bleu_score(
+                answers,
+                predictions,
+                n_gram=transformer.hparams["n_gram"],
+                smooth=transformer.hparams["smooth"],
+            ),
+            on_step=True,
+            on_epoch=True,
+        )
 
     @torch.no_grad()
     def on_validation_batch_end(
@@ -63,50 +87,75 @@ def on_validation_batch_end(
         out: dict,
         batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
         *args,
-        **kwargs
+        **kwargs,
     ) -> None:
         # logging validation metrics for each batch is unnecessary
         src, tgt_r, tgt_ids = batch
         transformer.log("validation/loss_epoch", out["loss"], on_epoch=True)
         transformer.log("validation/perplexity_epoch", torch.exp(out["loss"]), on_epoch=True)
         transformer.log("validation/accuracy_epoch", metricsF.accuracy(out["logits"], tgt_ids), on_epoch=True)
-        answers, predictions = self.on_any_batch_end("validation", transformer, src, tgt_r, tgt_ids,
-                                                     out['losses'].cpu().tolist())
-        transformer.log("validation/bleu_epoch", metricsF.bleu_score(answers, predictions), on_epoch=True)
+        answers, predictions = self.on_any_batch_end(
+            "validation", transformer, src, tgt_r, tgt_ids, out["losses"].cpu().tolist()
+        )
+        transformer.log(
+            "validation/bleu_epoch",
+            metricsF.bleu_score(
+                answers,
+                predictions,
+                n_gram=transformer.hparams["n_gram"],
+                smooth=transformer.hparams["smooth"],
+            ),
+            on_epoch=True,
+        )
 
     @torch.no_grad()
     def on_test_batch_end(
-            self,
-            trainer: Trainer,
-            transformer: Transformer,
-            out: dict,
-            batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
-            *args,
-            **kwargs
+        self,
+        trainer: Trainer,
+        transformer: Transformer,
+        out: dict,
+        batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
+        *args,
+        **kwargs,
     ) -> None:
         src, tgt_r, tgt_ids = batch
         transformer.log("test/loss_epoch", out["loss"], on_epoch=True)
         transformer.log("test/perplexity_epoch", torch.exp(out["loss"]), on_epoch=True)
-        transformer.log("test/accuracy_epoch", metricsF.accuracy(out["logits"], out['tgt_ids']), on_epoch=True)
-        answers, predictions = self.on_any_batch_end("test", transformer, src, tgt_r, tgt_ids,
-                                                     out['losses'].cpu().tolist())
-        transformer.log("test/bleu_epoch", metricsF.bleu_score(answers, predictions), on_epoch=True)
+        transformer.log(
+            "test/accuracy_epoch", metricsF.accuracy(out["logits"], out["tgt_ids"]), on_epoch=True
+        )
+        answers, predictions = self.on_any_batch_end(
+            "test", transformer, src, tgt_r, tgt_ids, out["losses"].cpu().tolist()
+        )
+        transformer.log(
+            "test/bleu_epoch",
+            metricsF.bleu_score(
+                answers,
+                predictions,
+                n_gram=transformer.hparams["n_gram"],
+                smooth=transformer.hparams["smooth"],
+            ),
+            on_epoch=True,
+        )
 
     # --- for logging on epoch end --- #
     @torch.no_grad()
     def on_any_epoch_end(self, key: str):
         """
         log BLEU scores, along with qualitative infos
         """
-        inputs = self.cache[key]['inputs']
-        predictions = self.cache[key]['predictions']
-        answers = self.cache[key]['answers']
-        losses = self.cache[key]['losses']
-        wandb.log({
-            f"{key}/examples":
-            wandb.Table(columns=["input", "prediction", "answer", "losses"],
-                        data=list(zip(inputs, predictions, answers, losses)))
-        })
+        inputs = self.cache[key]["inputs"]
+        predictions = self.cache[key]["predictions"]
+        answers = self.cache[key]["answers"]
+        losses = self.cache[key]["losses"]
+        wandb.log(
+            {
+                f"{key}/examples": wandb.Table(
+                    columns=["input", "prediction", "answer", "losses"],
+                    data=list(zip(inputs, predictions, answers, losses)),
+                )
+            }
+        )
 
     def on_train_epoch_end(self, *args, **kwargs) -> None:
         self.on_any_epoch_end("train")  # noqa

cleanformer/models/transformer.py

@@ -127,7 +127,7 @@ def training_step(self, batch: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
             "loss": losses.mean(dim=-1).mean(dim=-1),  # (N, L) -> (N,)  -> (1,)
             # --- for logging purposes --- #
             "losses": losses.mean(dim=-1).detach(),  # (N, L) -> (N,)
-            "logits": logits.detach()   # (N, |V|, L)
+            "logits": logits.detach(),  # (N, |V|, L)
         }
 
     @torch.no_grad()
@@ -145,21 +145,19 @@ def configure_optimizers(self) -> dict:
         optimizer = torch.optim.Adam(
             params=self.parameters(),
             lr=self.hparams["lr"],
-            betas=self.hparams['betas'],
-            eps=self.hparams['eps'],
-            weight_decay=self.hparams['weight_decay']
+            betas=self.hparams["betas"],
+            eps=self.hparams["eps"],
+            weight_decay=self.hparams["weight_decay"],
         )
         scheduler = ReduceLROnPlateau(
             optimizer,
             verbose=True,
-            mode=self.hparams['mode'],
-            patience=self.hparams['patience'],
-            cooldown=self.hparams['cooldown']
+            mode=self.hparams["mode"],
+            patience=self.hparams["patience"],
+            cooldown=self.hparams["cooldown"],
+            threshold=self.hparams['threshold']
         )
         return {
             "optimizer": optimizer,
-            "lr_scheduler": {
-                "scheduler": scheduler,
-                "monitor": self.hparams["monitor"]
-            }
+            "lr_scheduler": {"scheduler": scheduler, "monitor": self.hparams["monitor"]},
         }

cleanformer/translator.py

@@ -1,19 +1,18 @@
 from typing import Tuple
 from cleanformer.fetchers import fetch_config, fetch_transformer, fetch_tokenizer
-from cleanformer import preprocess as P   # noqa
+from cleanformer import preprocess as P  # noqa
 
 
 class Translator:
-
     def __init__(self):
-        config = fetch_config()['recommended']
+        config = fetch_config()["recommended"]
         self.transformer = fetch_transformer(config["transformer"]).eval()
         self.tokenizer = fetch_tokenizer(config["tokenizer"])
 
     def __call__(self, kor: str) -> Tuple[str, str]:
         x2y = [(kor, "")]
-        src = P.to_src(self.tokenizer, self.transformer.hparams['max_length'], x2y)
-        tgt_r = P.to_tgt_r(self.tokenizer, self.transformer.hparams['max_length'], x2y)
+        src = P.to_src(self.tokenizer, self.transformer.hparams["max_length"], x2y)
+        tgt_r = P.to_tgt_r(self.tokenizer, self.transformer.hparams["max_length"], x2y)
         pred_ids = self.transformer.infer(src, tgt_r).squeeze().tolist()  # (1, L) -> (L) -> list
         pred_ids = pred_ids[: pred_ids.index(self.tokenizer.eos_token_id)]  # noqa
         src_ids = src[0, 0].tolist()  # (1, 2, L) -> (L) -> list

config.yaml

@@ -3,15 +3,18 @@ transformer:
   hidden_size: 512
   ffn_size: 2048
   heads: 8
-  depth: 5
+  depth: 6
   max_length: 150
-  lr: 0.0001
   dropout: 0.1
   tokenizer: "tokenizer:v20"
+  # for bleu
+  n_gram: 4
+  smooth: 1
   # for dataloader
   seed: 410
   shuffle: true
   # for ADAM
+  lr: 0.0001
   eps: 0.000000001
   betas: [0.9, 0.98]
   weight_decay: 0.0005
@@ -20,10 +23,10 @@ transformer:
   mode: min
   patience: 3
   cooldown: 1
+  threshold: 0.001
 
 # --- config for building a tokenizer --- #
 tokenizer:
-  algorithm: wordpiece
   kor2eng: "kor2eng:v0"
   vocab_size: 16000
   pad: "[PAD]"

explore/explore_fetch_kor2eng.py

@@ -8,5 +8,5 @@ def main():
     print(len(test))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

explore/explore_torch._cross_entropy_reduce_none.py

@@ -1,15 +1,9 @@
-
-
 import torch
 from torch.nn import functional as F
 
-labels = torch.Tensor([[1, 2, 3],
-                       [2, 1, 3],
-                       [3, 2, 1]])
+labels = torch.Tensor([[1, 2, 3], [2, 1, 3], [3, 2, 1]])
 
-predictions = torch.Tensor([[10, 20, 30],
-                            [10, 20, 30],
-                            [10, 20, 30]])
+predictions = torch.Tensor([[10, 20, 30], [10, 20, 30], [10, 20, 30]])
 
 # this is not just a sum of each batch. The default reduce value is mean.
 print(F.cross_entropy(predictions, target=labels))  # (3, 3), (3, 3) -> (1,)

explore/explore_torch_cross_entropy_reduce_none_2d.py

@@ -1,12 +1,11 @@
-
 import torch
 from torch.nn import functional as F
 
 # (N, L)
 N = 3
 L = 5
 V = 10
-labels = torch.randint(low=0, high=V-1, size=(N, L))
+labels = torch.randint(low=0, high=V - 1, size=(N, L))
 
 # (N, C, L)
 preds = torch.rand(size=(N, V, L)).float()
@@ -21,4 +20,7 @@
 print(F.cross_entropy(preds, labels, reduction="none").mean(dim=-1))  # average over length
 print(F.cross_entropy(preds, labels, reduction="none").mean(dim=-1).mean(dim=-1))  # average over batch
 # they are essentially the same
-print(F.cross_entropy(preds, labels).item() == F.cross_entropy(preds, labels, reduction="none").mean(dim=-1).mean(dim=-1).item())
+print(
+    F.cross_entropy(preds, labels).item()
+    == F.cross_entropy(preds, labels, reduction="none").mean(dim=-1).mean(dim=-1).item()
+)

explore/explore_torch_detach.py

@@ -1,4 +1,3 @@
-
 import torch
 
 
@@ -19,5 +18,5 @@ def main():
     print(z.grad_fn.next_functions)
 
 
-if __name__ == '__main__':
-    main()
+if __name__ == "__main__":
+    main()

explore/explore_torchmetrics_bleu.py

@@ -0,0 +1,25 @@
+from torchmetrics import functional as F  # noqa
+
+
+# reference corpus must be an iterable (instances) of iterables (candidates) of iterables (tokens) of strings
+answers = [["I love you more than anyone else".split()], ["I hate you so much".split()]]
+# translation corpus must be an iterable (instances) of iterables (tokens) of strings
+predictions = ["I love you so much".split(), "I hate you so much".split()]
+
+# the scores should decrease as n_gram increases because the window of exact matches increases
+print(F.bleu_score(answers, predictions, n_gram=1))
+print(F.bleu_score(answers, predictions, n_gram=2))
+print(F.bleu_score(answers, predictions, n_gram=3))
+print(F.bleu_score(answers, predictions, n_gram=4))  # default
+print(F.bleu_score(answers, predictions, n_gram=5))
+# this should be zero because no predictions have length longer than 6
+# it is impossible to have 6-ngram matches (exact matches of length 6)
+print(F.bleu_score(answers, predictions, n_gram=6))
+print("after smoothing is applied")
+# why apply smoothing?: https://aclanthology.org/P04-1077.pdf
+print(F.bleu_score(answers, predictions, n_gram=1, smooth=True))
+print(F.bleu_score(answers, predictions, n_gram=2, smooth=True))
+print(F.bleu_score(answers, predictions, n_gram=3, smooth=True))
+print(F.bleu_score(answers, predictions, n_gram=4, smooth=True))
+print(F.bleu_score(answers, predictions, n_gram=5, smooth=True))
+print(F.bleu_score(answers, predictions, n_gram=6, smooth=True))

main_build_tokenizer.py

@@ -30,15 +30,15 @@
 tokenizer.pre_tokenizer = pre_tokenizers.Sequence([Whitespace(), Digits(), Punctuation()])  # noqa
 tokenizer.normalizer = normalizers.Sequence([Lowercase()])  # noqa
 # --- prepare the data --- #
-kor2eng_train, kor2eng_val, kor2eng_test = fetch_kor2eng(config['kor2eng'])
+train, val, test = fetch_kor2eng(config["kor2eng"])
 # chaining two generators;  https://stackoverflow.com/a/3211047
 iterator = chain(
-    (kor for kor, _ in kor2eng_train),
-    (eng for _, eng in kor2eng_train),
-    (kor for kor, _ in kor2eng_val),
-    (eng for _, eng in kor2eng_val),
-    (kor for kor, _ in kor2eng_test),
-    (eng for _, eng in kor2eng_test),
+    (kor for kor, _ in train),
+    (eng for _, eng in train),
+    (kor for kor, _ in val),
+    (eng for _, eng in val),
+    (kor for kor, _ in test),
+    (eng for _, eng in test),
 )
 # --- train the tokenizer --- #
 tokenizer.train_from_iterator(iterator, trainer=trainer)

main_deploy.py

@@ -1,3 +1,3 @@
 """
 Deploy a pre-trained transformer with fastapi.
-"""
+"""