some tokenizers do not have additional_special_tokens_ids attribute (#…

…5642) (#5648)

Signed-off-by: arendu <adithya.r@gmail.com>

Signed-off-by: arendu <adithya.r@gmail.com>

Signed-off-by: arendu <adithya.r@gmail.com>
Co-authored-by: Adi Renduchintala <108822655+arendu@users.noreply.github.com>
Co-authored-by: Eric Harper <complex451@gmail.com>
Signed-off-by: Elena Rastorgueva <erastorgueva@nvidia.com>

nemo/collections/nlp/models/language_modeling/megatron_t5_adapter_model.py

@@ -128,9 +128,13 @@ def validation_step(self, batch, batch_idx, inference=False):
                 idx = pred.index(self.tokenizer.eos_id)
                 pred = pred[:idx]
 
-            pred = [id for id in pred if id not in self.tokenizer.tokenizer.additional_special_tokens_ids]
-            label = [id for id in label if id not in self.tokenizer.tokenizer.additional_special_tokens_ids]
-            enc_input = [id for id in enc_input if id not in self.tokenizer.tokenizer.additional_special_tokens_ids]
+            additional_special_tokens_ids = []
+            if hasattr(self.tokenizer.tokenizer, "additional_special_tokens_ids"):
+                additional_special_tokens_ids = self.tokenizer.tokenizer.additional_special_tokens_ids
+
+            pred = [id for id in pred if id not in additional_special_tokens_ids]
+            label = [id for id in label if id not in additional_special_tokens_ids]
+            enc_input = [id for id in enc_input if id not in additional_special_tokens_ids]
 
             pred = self.tokenizer.ids_to_text(pred)
             label = self.tokenizer.ids_to_text(label)