Fix typo

Author: mainpyp

src/modeling/diffusion/gaussian_diffusion.py

@@ -274,7 +274,7 @@ def training_losses(self, model, x_start_never_used, t, model_kwargs=None, noise
         assert (
             model_output.shape == target.shape == x_start.shape
         ), f"model_output.shape: {model_output.shape}, target.shape: {target.shape}, x_start.shape: {x_start.shape}"
-        # the usual diffusion loss
+        # the usual diffusion loss we use simple MSE (target minus output squared)
         terms["mse"] = mean_flat((target - model_output) ** 2)
         # print( terms["mse"])
         model_out_x_start = self.x0_helper(model_output, x_t, t)["pred_xstart"]

src/train_infer/train_loop.py

@@ -255,7 +255,7 @@ def forward_backward(self, batch, cond):
             last_batch = (i + self.microbatch) >= batch_size
             curr_batch_size = micro_cond[list(micro_cond.keys())[0]].shape[0]
             t, weights = self.schedule_sampler.sample(curr_batch_size, dist_util.dev())
-            # print(micro_cond.keys())
+            print(f"Micro cond keys (in forward_backward): {micro_cond.keys()}")
             compute_losses = functools.partial(
                 self.diffusion.training_losses,
                 self.ddp_model,  # this is the transformer model in DDP mode

src/utils/data_utils_sentencepiece.py

@@ -53,7 +53,7 @@ def read_data(self):
 
         def tokenization(example):
             return self.tokenizer(example["text"],
-                                  max_lenght=512,
+                                  max_length=512,
                                   padding=True,
                                   truncation=True)