Extend Nemo AutoTokenizer for TRT-LLM evaluation usage

Signed-off-by: Jan Lasek <janek.lasek@gmail.com>

nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

@@ -218,6 +218,18 @@ def ids_to_text(self, ids):
         text = self.tokens_to_text(tokens_clean)
         return text
 
+    def encode(self, *args, **kwargs):
+        return self.tokenizer.encode(*args, **kwargs)
+
+    def batch_encode_plus(self, *args, **kwargs):
+        return self.tokenizer.batch_encode_plus(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def batch_decode(self, *args, **kwargs):
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
     @property
     def vocab(self):
         id2vocab = {v: k for k, v in self.tokenizer.vocab.items()}
@@ -241,6 +253,18 @@ def eos_id(self):
             return None
         return self.tokens_to_ids([getattr(self, 'eos_token')])[0]
 
+    @property
+    def pad_token_id(self):
+        return self.pad_id
+
+    @pad_token_id.setter
+    def pad_token_id(self, value):
+        self.pad_token = self.ids_to_tokens(value)
+
+    @property
+    def eos_token_id(self):
+        return self.eos_id
+
     @property
     def eod(self):
         """Returns EOS token id. Exact copy of the eos_id function. Required for megatron-core."""