Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

S2S + M2M100 should be available in tokenization_auto #10657

Merged
merged 3 commits into from
Mar 11, 2021
Merged

S2S + M2M100 should be available in tokenization_auto #10657

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f32fea0
S2S + M2M100 should be available in tokenization_auto
LysandreJik Mar 11, 2021
e672f07
Requires sentencepiece
LysandreJik Mar 11, 2021
d44e28b
SentencePiece for S2T as well :)
LysandreJik Mar 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/transformers/models/auto/tokenization_auto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
from ..rag.tokenization_rag import RagTokenizer
from ..retribert.tokenization_retribert import RetriBertTokenizer
from ..roberta.tokenization_roberta import RobertaTokenizer
from ..speech_to_text import Speech2TextTokenizer
from ..squeezebert.tokenization_squeezebert import SqueezeBertTokenizer
from ..tapas.tokenization_tapas import TapasTokenizer
from ..transfo_xl.tokenization_transfo_xl import TransfoXLTokenizer
Expand Down Expand Up @@ -80,6 +81,7 @@
LEDConfig,
LongformerConfig,
LxmertConfig,
M2M100Config,
MarianConfig,
MBartConfig,
MobileBertConfig,
Expand All @@ -92,6 +94,7 @@
ReformerConfig,
RetriBertConfig,
RobertaConfig,
Speech2TextConfig,
SqueezeBertConfig,
T5Config,
TapasConfig,
Expand All @@ -111,6 +114,7 @@
from ..bert_generation.tokenization_bert_generation import BertGenerationTokenizer
from ..camembert.tokenization_camembert import CamembertTokenizer
from ..deberta_v2.tokenization_deberta_v2 import DebertaV2Tokenizer
from ..m2m_100 import M2M100Tokenizer
from ..marian.tokenization_marian import MarianTokenizer
from ..mbart.tokenization_mbart import MBartTokenizer
from ..mt5 import MT5Tokenizer
Expand All @@ -135,6 +139,7 @@
XLMRobertaTokenizer = None
XLNetTokenizer = None
XLMProphetNetTokenizer = None
M2M100Tokenizer = None

if is_tokenizers_available():
from ..albert.tokenization_albert_fast import AlbertTokenizerFast
Expand Down Expand Up @@ -240,6 +245,8 @@
(DebertaV2Config, (DebertaV2Tokenizer, None)),
(RagConfig, (RagTokenizer, None)),
(XLMProphetNetConfig, (XLMProphetNetTokenizer, None)),
(Speech2TextConfig, (Speech2TextTokenizer, None)),
(M2M100Config, (M2M100Tokenizer, None)),
(ProphetNetConfig, (ProphetNetTokenizer, None)),
(MPNetConfig, (MPNetTokenizer, MPNetTokenizerFast)),
(TapasConfig, (TapasTokenizer, None)),
Expand Down