Merge pull request #74 from AntoineSimoulin/patch-1

🐛 fix trunctation when preparing bert data
MAIF · Dec 3, 2020 · 53d7f99 · 53d7f99
2 parents 9bccac9 + b24ab2e
commit 53d7f99
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/melusine/models/train.py b/melusine/models/train.py
@@ -449,7 +449,7 @@ def _prepare_bert_sequences(self, X):
         else:
             sequence = X[self.text_input_column].values.tolist()
         seqs = self.tokenizer.batch_encode_plus(
-            sequence, max_length=self.seq_size, padding="max_length"
+            sequence, max_length=self.seq_size, padding="max_length", truncation=True
         )
 
         return (