{"payload":{"allShortcutsEnabled":false,"fileTree":{"src/transformers/models/mbart50":{"items":[{"name":"__init__.py","path":"src/transformers/models/mbart50/__init__.py","contentType":"file"},{"name":"tokenization_mbart50.py","path":"src/transformers/models/mbart50/tokenization_mbart50.py","contentType":"file"},{"name":"tokenization_mbart50_fast.py","path":"src/transformers/models/mbart50/tokenization_mbart50_fast.py","contentType":"file"}],"totalCount":3},"src/transformers/models":{"items":[{"name":"albert","path":"src/transformers/models/albert","contentType":"directory"},{"name":"align","path":"src/transformers/models/align","contentType":"directory"},{"name":"altclip","path":"src/transformers/models/altclip","contentType":"directory"},{"name":"audio_spectrogram_transformer","path":"src/transformers/models/audio_spectrogram_transformer","contentType":"directory"},{"name":"auto","path":"src/transformers/models/auto","contentType":"directory"},{"name":"autoformer","path":"src/transformers/models/autoformer","contentType":"directory"},{"name":"bark","path":"src/transformers/models/bark","contentType":"directory"},{"name":"bart","path":"src/transformers/models/bart","contentType":"directory"},{"name":"barthez","path":"src/transformers/models/barthez","contentType":"directory"},{"name":"bartpho","path":"src/transformers/models/bartpho","contentType":"directory"},{"name":"beit","path":"src/transformers/models/beit","contentType":"directory"},{"name":"bert","path":"src/transformers/models/bert","contentType":"directory"},{"name":"bert_generation","path":"src/transformers/models/bert_generation","contentType":"directory"},{"name":"bert_japanese","path":"src/transformers/models/bert_japanese","contentType":"directory"},{"name":"bertweet","path":"src/transformers/models/bertweet","contentType":"directory"},{"name":"big_bird","path":"src/transformers/models/big_bird","contentType":"directory"},{"name":"bigbird_pegasus","path":"src/transformers/models/bigbird_pegasus","contentType":"directory"},{"name":"biogpt","path":"src/transformers/models/biogpt","contentType":"directory"},{"name":"bit","path":"src/transformers/models/bit","contentType":"directory"},{"name":"blenderbot","path":"src/transformers/models/blenderbot","contentType":"directory"},{"name":"blenderbot_small","path":"src/transformers/models/blenderbot_small","contentType":"directory"},{"name":"blip","path":"src/transformers/models/blip","contentType":"directory"},{"name":"blip_2","path":"src/transformers/models/blip_2","contentType":"directory"},{"name":"bloom","path":"src/transformers/models/bloom","contentType":"directory"},{"name":"bridgetower","path":"src/transformers/models/bridgetower","contentType":"directory"},{"name":"bros","path":"src/transformers/models/bros","contentType":"directory"},{"name":"byt5","path":"src/transformers/models/byt5","contentType":"directory"},{"name":"camembert","path":"src/transformers/models/camembert","contentType":"directory"},{"name":"canine","path":"src/transformers/models/canine","contentType":"directory"},{"name":"chinese_clip","path":"src/transformers/models/chinese_clip","contentType":"directory"},{"name":"clap","path":"src/transformers/models/clap","contentType":"directory"},{"name":"clip","path":"src/transformers/models/clip","contentType":"directory"},{"name":"clipseg","path":"src/transformers/models/clipseg","contentType":"directory"},{"name":"clvp","path":"src/transformers/models/clvp","contentType":"directory"},{"name":"code_llama","path":"src/transformers/models/code_llama","contentType":"directory"},{"name":"codegen","path":"src/transformers/models/codegen","contentType":"directory"},{"name":"cohere","path":"src/transformers/models/cohere","contentType":"directory"},{"name":"conditional_detr","path":"src/transformers/models/conditional_detr","contentType":"directory"},{"name":"convbert","path":"src/transformers/models/convbert","contentType":"directory"},{"name":"convnext","path":"src/transformers/models/convnext","contentType":"directory"},{"name":"convnextv2","path":"src/transformers/models/convnextv2","contentType":"directory"},{"name":"cpm","path":"src/transformers/models/cpm","contentType":"directory"},{"name":"cpmant","path":"src/transformers/models/cpmant","contentType":"directory"},{"name":"ctrl","path":"src/transformers/models/ctrl","contentType":"directory"},{"name":"cvt","path":"src/transformers/models/cvt","contentType":"directory"},{"name":"data2vec","path":"src/transformers/models/data2vec","contentType":"directory"},{"name":"deberta","path":"src/transformers/models/deberta","contentType":"directory"},{"name":"deberta_v2","path":"src/transformers/models/deberta_v2","contentType":"directory"},{"name":"decision_transformer","path":"src/transformers/models/decision_transformer","contentType":"directory"},{"name":"deformable_detr","path":"src/transformers/models/deformable_detr","contentType":"directory"},{"name":"deit","path":"src/transformers/models/deit","contentType":"directory"},{"name":"deprecated","path":"src/transformers/models/deprecated","contentType":"directory"},{"name":"depth_anything","path":"src/transformers/models/depth_anything","contentType":"directory"},{"name":"deta","path":"src/transformers/models/deta","contentType":"directory"},{"name":"detr","path":"src/transformers/models/detr","contentType":"directory"},{"name":"dialogpt","path":"src/transformers/models/dialogpt","contentType":"directory"},{"name":"dinat","path":"src/transformers/models/dinat","contentType":"directory"},{"name":"dinov2","path":"src/transformers/models/dinov2","contentType":"directory"},{"name":"distilbert","path":"src/transformers/models/distilbert","contentType":"directory"},{"name":"dit","path":"src/transformers/models/dit","contentType":"directory"},{"name":"donut","path":"src/transformers/models/donut","contentType":"directory"},{"name":"dpr","path":"src/transformers/models/dpr","contentType":"directory"},{"name":"dpt","path":"src/transformers/models/dpt","contentType":"directory"},{"name":"efficientformer","path":"src/transformers/models/efficientformer","contentType":"directory"},{"name":"efficientnet","path":"src/transformers/models/efficientnet","contentType":"directory"},{"name":"electra","path":"src/transformers/models/electra","contentType":"directory"},{"name":"encodec","path":"src/transformers/models/encodec","contentType":"directory"},{"name":"encoder_decoder","path":"src/transformers/models/encoder_decoder","contentType":"directory"},{"name":"ernie","path":"src/transformers/models/ernie","contentType":"directory"},{"name":"ernie_m","path":"src/transformers/models/ernie_m","contentType":"directory"},{"name":"esm","path":"src/transformers/models/esm","contentType":"directory"},{"name":"falcon","path":"src/transformers/models/falcon","contentType":"directory"},{"name":"fastspeech2_conformer","path":"src/transformers/models/fastspeech2_conformer","contentType":"directory"},{"name":"flaubert","path":"src/transformers/models/flaubert","contentType":"directory"},{"name":"flava","path":"src/transformers/models/flava","contentType":"directory"},{"name":"fnet","path":"src/transformers/models/fnet","contentType":"directory"},{"name":"focalnet","path":"src/transformers/models/focalnet","contentType":"directory"},{"name":"fsmt","path":"src/transformers/models/fsmt","contentType":"directory"},{"name":"funnel","path":"src/transformers/models/funnel","contentType":"directory"},{"name":"fuyu","path":"src/transformers/models/fuyu","contentType":"directory"},{"name":"gemma","path":"src/transformers/models/gemma","contentType":"directory"},{"name":"git","path":"src/transformers/models/git","contentType":"directory"},{"name":"glpn","path":"src/transformers/models/glpn","contentType":"directory"},{"name":"gpt2","path":"src/transformers/models/gpt2","contentType":"directory"},{"name":"gpt_bigcode","path":"src/transformers/models/gpt_bigcode","contentType":"directory"},{"name":"gpt_neo","path":"src/transformers/models/gpt_neo","contentType":"directory"},{"name":"gpt_neox","path":"src/transformers/models/gpt_neox","contentType":"directory"},{"name":"gpt_neox_japanese","path":"src/transformers/models/gpt_neox_japanese","contentType":"directory"},{"name":"gpt_sw3","path":"src/transformers/models/gpt_sw3","contentType":"directory"},{"name":"gptj","path":"src/transformers/models/gptj","contentType":"directory"},{"name":"gptsan_japanese","path":"src/transformers/models/gptsan_japanese","contentType":"directory"},{"name":"graphormer","path":"src/transformers/models/graphormer","contentType":"directory"},{"name":"groupvit","path":"src/transformers/models/groupvit","contentType":"directory"},{"name":"herbert","path":"src/transformers/models/herbert","contentType":"directory"},{"name":"hubert","path":"src/transformers/models/hubert","contentType":"directory"},{"name":"ibert","path":"src/transformers/models/ibert","contentType":"directory"},{"name":"idefics","path":"src/transformers/models/idefics","contentType":"directory"},{"name":"imagegpt","path":"src/transformers/models/imagegpt","contentType":"directory"},{"name":"informer","path":"src/transformers/models/informer","contentType":"directory"},{"name":"instructblip","path":"src/transformers/models/instructblip","contentType":"directory"},{"name":"jukebox","path":"src/transformers/models/jukebox","contentType":"directory"},{"name":"kosmos2","path":"src/transformers/models/kosmos2","contentType":"directory"},{"name":"layoutlm","path":"src/transformers/models/layoutlm","contentType":"directory"},{"name":"layoutlmv2","path":"src/transformers/models/layoutlmv2","contentType":"directory"},{"name":"layoutlmv3","path":"src/transformers/models/layoutlmv3","contentType":"directory"},{"name":"layoutxlm","path":"src/transformers/models/layoutxlm","contentType":"directory"},{"name":"led","path":"src/transformers/models/led","contentType":"directory"},{"name":"levit","path":"src/transformers/models/levit","contentType":"directory"},{"name":"lilt","path":"src/transformers/models/lilt","contentType":"directory"},{"name":"llama","path":"src/transformers/models/llama","contentType":"directory"},{"name":"llava","path":"src/transformers/models/llava","contentType":"directory"},{"name":"longformer","path":"src/transformers/models/longformer","contentType":"directory"},{"name":"longt5","path":"src/transformers/models/longt5","contentType":"directory"},{"name":"luke","path":"src/transformers/models/luke","contentType":"directory"},{"name":"lxmert","path":"src/transformers/models/lxmert","contentType":"directory"},{"name":"m2m_100","path":"src/transformers/models/m2m_100","contentType":"directory"},{"name":"mamba","path":"src/transformers/models/mamba","contentType":"directory"},{"name":"marian","path":"src/transformers/models/marian","contentType":"directory"},{"name":"markuplm","path":"src/transformers/models/markuplm","contentType":"directory"},{"name":"mask2former","path":"src/transformers/models/mask2former","contentType":"directory"},{"name":"maskformer","path":"src/transformers/models/maskformer","contentType":"directory"},{"name":"mbart","path":"src/transformers/models/mbart","contentType":"directory"},{"name":"mbart50","path":"src/transformers/models/mbart50","contentType":"directory"},{"name":"mega","path":"src/transformers/models/mega","contentType":"directory"},{"name":"megatron_bert","path":"src/transformers/models/megatron_bert","contentType":"directory"},{"name":"megatron_gpt2","path":"src/transformers/models/megatron_gpt2","contentType":"directory"},{"name":"mgp_str","path":"src/transformers/models/mgp_str","contentType":"directory"},{"name":"mistral","path":"src/transformers/models/mistral","contentType":"directory"},{"name":"mixtral","path":"src/transformers/models/mixtral","contentType":"directory"},{"name":"mluke","path":"src/transformers/models/mluke","contentType":"directory"},{"name":"mobilebert","path":"src/transformers/models/mobilebert","contentType":"directory"},{"name":"mobilenet_v1","path":"src/transformers/models/mobilenet_v1","contentType":"directory"},{"name":"mobilenet_v2","path":"src/transformers/models/mobilenet_v2","contentType":"directory"},{"name":"mobilevit","path":"src/transformers/models/mobilevit","contentType":"directory"},{"name":"mobilevitv2","path":"src/transformers/models/mobilevitv2","contentType":"directory"},{"name":"mpnet","path":"src/transformers/models/mpnet","contentType":"directory"},{"name":"mpt","path":"src/transformers/models/mpt","contentType":"directory"},{"name":"mra","path":"src/transformers/models/mra","contentType":"directory"},{"name":"mt5","path":"src/transformers/models/mt5","contentType":"directory"},{"name":"musicgen","path":"src/transformers/models/musicgen","contentType":"directory"},{"name":"musicgen_melody","path":"src/transformers/models/musicgen_melody","contentType":"directory"},{"name":"mvp","path":"src/transformers/models/mvp","contentType":"directory"},{"name":"nat","path":"src/transformers/models/nat","contentType":"directory"},{"name":"nezha","path":"src/transformers/models/nezha","contentType":"directory"},{"name":"nllb","path":"src/transformers/models/nllb","contentType":"directory"},{"name":"nllb_moe","path":"src/transformers/models/nllb_moe","contentType":"directory"},{"name":"nougat","path":"src/transformers/models/nougat","contentType":"directory"},{"name":"nystromformer","path":"src/transformers/models/nystromformer","contentType":"directory"},{"name":"oneformer","path":"src/transformers/models/oneformer","contentType":"directory"},{"name":"openai","path":"src/transformers/models/openai","contentType":"directory"},{"name":"opt","path":"src/transformers/models/opt","contentType":"directory"},{"name":"owlv2","path":"src/transformers/models/owlv2","contentType":"directory"},{"name":"owlvit","path":"src/transformers/models/owlvit","contentType":"directory"},{"name":"patchtsmixer","path":"src/transformers/models/patchtsmixer","contentType":"directory"},{"name":"patchtst","path":"src/transformers/models/patchtst","contentType":"directory"},{"name":"pegasus","path":"src/transformers/models/pegasus","contentType":"directory"},{"name":"pegasus_x","path":"src/transformers/models/pegasus_x","contentType":"directory"},{"name":"perceiver","path":"src/transformers/models/perceiver","contentType":"directory"},{"name":"persimmon","path":"src/transformers/models/persimmon","contentType":"directory"},{"name":"phi","path":"src/transformers/models/phi","contentType":"directory"},{"name":"phobert","path":"src/transformers/models/phobert","contentType":"directory"},{"name":"pix2struct","path":"src/transformers/models/pix2struct","contentType":"directory"},{"name":"plbart","path":"src/transformers/models/plbart","contentType":"directory"},{"name":"poolformer","path":"src/transformers/models/poolformer","contentType":"directory"},{"name":"pop2piano","path":"src/transformers/models/pop2piano","contentType":"directory"},{"name":"prophetnet","path":"src/transformers/models/prophetnet","contentType":"directory"},{"name":"pvt","path":"src/transformers/models/pvt","contentType":"directory"},{"name":"pvt_v2","path":"src/transformers/models/pvt_v2","contentType":"directory"},{"name":"qdqbert","path":"src/transformers/models/qdqbert","contentType":"directory"},{"name":"qwen2","path":"src/transformers/models/qwen2","contentType":"directory"},{"name":"rag","path":"src/transformers/models/rag","contentType":"directory"},{"name":"realm","path":"src/transformers/models/realm","contentType":"directory"},{"name":"reformer","path":"src/transformers/models/reformer","contentType":"directory"},{"name":"regnet","path":"src/transformers/models/regnet","contentType":"directory"},{"name":"rembert","path":"src/transformers/models/rembert","contentType":"directory"},{"name":"resnet","path":"src/transformers/models/resnet","contentType":"directory"},{"name":"roberta","path":"src/transformers/models/roberta","contentType":"directory"},{"name":"roberta_prelayernorm","path":"src/transformers/models/roberta_prelayernorm","contentType":"directory"},{"name":"roc_bert","path":"src/transformers/models/roc_bert","contentType":"directory"},{"name":"roformer","path":"src/transformers/models/roformer","contentType":"directory"},{"name":"rwkv","path":"src/transformers/models/rwkv","contentType":"directory"},{"name":"sam","path":"src/transformers/models/sam","contentType":"directory"},{"name":"seamless_m4t","path":"src/transformers/models/seamless_m4t","contentType":"directory"},{"name":"seamless_m4t_v2","path":"src/transformers/models/seamless_m4t_v2","contentType":"directory"},{"name":"segformer","path":"src/transformers/models/segformer","contentType":"directory"},{"name":"seggpt","path":"src/transformers/models/seggpt","contentType":"directory"},{"name":"sew","path":"src/transformers/models/sew","contentType":"directory"},{"name":"sew_d","path":"src/transformers/models/sew_d","contentType":"directory"},{"name":"siglip","path":"src/transformers/models/siglip","contentType":"directory"},{"name":"speech_encoder_decoder","path":"src/transformers/models/speech_encoder_decoder","contentType":"directory"},{"name":"speech_to_text","path":"src/transformers/models/speech_to_text","contentType":"directory"},{"name":"speech_to_text_2","path":"src/transformers/models/speech_to_text_2","contentType":"directory"},{"name":"speecht5","path":"src/transformers/models/speecht5","contentType":"directory"},{"name":"splinter","path":"src/transformers/models/splinter","contentType":"directory"},{"name":"squeezebert","path":"src/transformers/models/squeezebert","contentType":"directory"},{"name":"stablelm","path":"src/transformers/models/stablelm","contentType":"directory"},{"name":"starcoder2","path":"src/transformers/models/starcoder2","contentType":"directory"},{"name":"swiftformer","path":"src/transformers/models/swiftformer","contentType":"directory"},{"name":"swin","path":"src/transformers/models/swin","contentType":"directory"},{"name":"swin2sr","path":"src/transformers/models/swin2sr","contentType":"directory"},{"name":"swinv2","path":"src/transformers/models/swinv2","contentType":"directory"},{"name":"switch_transformers","path":"src/transformers/models/switch_transformers","contentType":"directory"},{"name":"t5","path":"src/transformers/models/t5","contentType":"directory"},{"name":"table_transformer","path":"src/transformers/models/table_transformer","contentType":"directory"},{"name":"tapas","path":"src/transformers/models/tapas","contentType":"directory"},{"name":"time_series_transformer","path":"src/transformers/models/time_series_transformer","contentType":"directory"},{"name":"timesformer","path":"src/transformers/models/timesformer","contentType":"directory"},{"name":"timm_backbone","path":"src/transformers/models/timm_backbone","contentType":"directory"},{"name":"trocr","path":"src/transformers/models/trocr","contentType":"directory"},{"name":"tvlt","path":"src/transformers/models/tvlt","contentType":"directory"},{"name":"tvp","path":"src/transformers/models/tvp","contentType":"directory"},{"name":"udop","path":"src/transformers/models/udop","contentType":"directory"},{"name":"umt5","path":"src/transformers/models/umt5","contentType":"directory"},{"name":"unispeech","path":"src/transformers/models/unispeech","contentType":"directory"},{"name":"unispeech_sat","path":"src/transformers/models/unispeech_sat","contentType":"directory"},{"name":"univnet","path":"src/transformers/models/univnet","contentType":"directory"},{"name":"upernet","path":"src/transformers/models/upernet","contentType":"directory"},{"name":"videomae","path":"src/transformers/models/videomae","contentType":"directory"},{"name":"vilt","path":"src/transformers/models/vilt","contentType":"directory"},{"name":"vipllava","path":"src/transformers/models/vipllava","contentType":"directory"},{"name":"vision_encoder_decoder","path":"src/transformers/models/vision_encoder_decoder","contentType":"directory"},{"name":"vision_text_dual_encoder","path":"src/transformers/models/vision_text_dual_encoder","contentType":"directory"},{"name":"visual_bert","path":"src/transformers/models/visual_bert","contentType":"directory"},{"name":"vit","path":"src/transformers/models/vit","contentType":"directory"},{"name":"vit_hybrid","path":"src/transformers/models/vit_hybrid","contentType":"directory"},{"name":"vit_mae","path":"src/transformers/models/vit_mae","contentType":"directory"},{"name":"vit_msn","path":"src/transformers/models/vit_msn","contentType":"directory"},{"name":"vitdet","path":"src/transformers/models/vitdet","contentType":"directory"},{"name":"vitmatte","path":"src/transformers/models/vitmatte","contentType":"directory"},{"name":"vits","path":"src/transformers/models/vits","contentType":"directory"},{"name":"vivit","path":"src/transformers/models/vivit","contentType":"directory"},{"name":"wav2vec2","path":"src/transformers/models/wav2vec2","contentType":"directory"},{"name":"wav2vec2_bert","path":"src/transformers/models/wav2vec2_bert","contentType":"directory"},{"name":"wav2vec2_conformer","path":"src/transformers/models/wav2vec2_conformer","contentType":"directory"},{"name":"wav2vec2_phoneme","path":"src/transformers/models/wav2vec2_phoneme","contentType":"directory"},{"name":"wav2vec2_with_lm","path":"src/transformers/models/wav2vec2_with_lm","contentType":"directory"},{"name":"wavlm","path":"src/transformers/models/wavlm","contentType":"directory"},{"name":"whisper","path":"src/transformers/models/whisper","contentType":"directory"},{"name":"x_clip","path":"src/transformers/models/x_clip","contentType":"directory"},{"name":"xglm","path":"src/transformers/models/xglm","contentType":"directory"},{"name":"xlm","path":"src/transformers/models/xlm","contentType":"directory"},{"name":"xlm_prophetnet","path":"src/transformers/models/xlm_prophetnet","contentType":"directory"},{"name":"xlm_roberta","path":"src/transformers/models/xlm_roberta","contentType":"directory"},{"name":"xlm_roberta_xl","path":"src/transformers/models/xlm_roberta_xl","contentType":"directory"},{"name":"xlnet","path":"src/transformers/models/xlnet","contentType":"directory"},{"name":"xmod","path":"src/transformers/models/xmod","contentType":"directory"},{"name":"yolos","path":"src/transformers/models/yolos","contentType":"directory"},{"name":"yoso","path":"src/transformers/models/yoso","contentType":"directory"},{"name":"__init__.py","path":"src/transformers/models/__init__.py","contentType":"file"}],"totalCount":249},"src/transformers":{"items":[{"name":"benchmark","path":"src/transformers/benchmark","contentType":"directory"},{"name":"commands","path":"src/transformers/commands","contentType":"directory"},{"name":"data","path":"src/transformers/data","contentType":"directory"},{"name":"generation","path":"src/transformers/generation","contentType":"directory"},{"name":"integrations","path":"src/transformers/integrations","contentType":"directory"},{"name":"kernels","path":"src/transformers/kernels","contentType":"directory"},{"name":"models","path":"src/transformers/models","contentType":"directory"},{"name":"onnx","path":"src/transformers/onnx","contentType":"directory"},{"name":"pipelines","path":"src/transformers/pipelines","contentType":"directory"},{"name":"quantizers","path":"src/transformers/quantizers","contentType":"directory"},{"name":"sagemaker","path":"src/transformers/sagemaker","contentType":"directory"},{"name":"tools","path":"src/transformers/tools","contentType":"directory"},{"name":"utils","path":"src/transformers/utils","contentType":"directory"},{"name":"__init__.py","path":"src/transformers/__init__.py","contentType":"file"},{"name":"activations.py","path":"src/transformers/activations.py","contentType":"file"},{"name":"activations_tf.py","path":"src/transformers/activations_tf.py","contentType":"file"},{"name":"audio_utils.py","path":"src/transformers/audio_utils.py","contentType":"file"},{"name":"cache_utils.py","path":"src/transformers/cache_utils.py","contentType":"file"},{"name":"configuration_utils.py","path":"src/transformers/configuration_utils.py","contentType":"file"},{"name":"convert_graph_to_onnx.py","path":"src/transformers/convert_graph_to_onnx.py","contentType":"file"},{"name":"convert_pytorch_checkpoint_to_tf2.py","path":"src/transformers/convert_pytorch_checkpoint_to_tf2.py","contentType":"file"},{"name":"convert_slow_tokenizer.py","path":"src/transformers/convert_slow_tokenizer.py","contentType":"file"},{"name":"convert_slow_tokenizers_checkpoints_to_fast.py","path":"src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py","contentType":"file"},{"name":"convert_tf_hub_seq_to_seq_bert_to_pytorch.py","path":"src/transformers/convert_tf_hub_seq_to_seq_bert_to_pytorch.py","contentType":"file"},{"name":"debug_utils.py","path":"src/transformers/debug_utils.py","contentType":"file"},{"name":"deepspeed.py","path":"src/transformers/deepspeed.py","contentType":"file"},{"name":"dependency_versions_check.py","path":"src/transformers/dependency_versions_check.py","contentType":"file"},{"name":"dependency_versions_table.py","path":"src/transformers/dependency_versions_table.py","contentType":"file"},{"name":"dynamic_module_utils.py","path":"src/transformers/dynamic_module_utils.py","contentType":"file"},{"name":"feature_extraction_sequence_utils.py","path":"src/transformers/feature_extraction_sequence_utils.py","contentType":"file"},{"name":"feature_extraction_utils.py","path":"src/transformers/feature_extraction_utils.py","contentType":"file"},{"name":"file_utils.py","path":"src/transformers/file_utils.py","contentType":"file"},{"name":"generation_flax_utils.py","path":"src/transformers/generation_flax_utils.py","contentType":"file"},{"name":"generation_tf_utils.py","path":"src/transformers/generation_tf_utils.py","contentType":"file"},{"name":"generation_utils.py","path":"src/transformers/generation_utils.py","contentType":"file"},{"name":"hf_argparser.py","path":"src/transformers/hf_argparser.py","contentType":"file"},{"name":"hyperparameter_search.py","path":"src/transformers/hyperparameter_search.py","contentType":"file"},{"name":"image_processing_utils.py","path":"src/transformers/image_processing_utils.py","contentType":"file"},{"name":"image_transforms.py","path":"src/transformers/image_transforms.py","contentType":"file"},{"name":"image_utils.py","path":"src/transformers/image_utils.py","contentType":"file"},{"name":"keras_callbacks.py","path":"src/transformers/keras_callbacks.py","contentType":"file"},{"name":"modelcard.py","path":"src/transformers/modelcard.py","contentType":"file"},{"name":"modeling_attn_mask_utils.py","path":"src/transformers/modeling_attn_mask_utils.py","contentType":"file"},{"name":"modeling_flax_outputs.py","path":"src/transformers/modeling_flax_outputs.py","contentType":"file"},{"name":"modeling_flax_pytorch_utils.py","path":"src/transformers/modeling_flax_pytorch_utils.py","contentType":"file"},{"name":"modeling_flax_utils.py","path":"src/transformers/modeling_flax_utils.py","contentType":"file"},{"name":"modeling_outputs.py","path":"src/transformers/modeling_outputs.py","contentType":"file"},{"name":"modeling_tf_outputs.py","path":"src/transformers/modeling_tf_outputs.py","contentType":"file"},{"name":"modeling_tf_pytorch_utils.py","path":"src/transformers/modeling_tf_pytorch_utils.py","contentType":"file"},{"name":"modeling_tf_utils.py","path":"src/transformers/modeling_tf_utils.py","contentType":"file"},{"name":"modeling_utils.py","path":"src/transformers/modeling_utils.py","contentType":"file"},{"name":"optimization.py","path":"src/transformers/optimization.py","contentType":"file"},{"name":"optimization_tf.py","path":"src/transformers/optimization_tf.py","contentType":"file"},{"name":"processing_utils.py","path":"src/transformers/processing_utils.py","contentType":"file"},{"name":"pytorch_utils.py","path":"src/transformers/pytorch_utils.py","contentType":"file"},{"name":"safetensors_conversion.py","path":"src/transformers/safetensors_conversion.py","contentType":"file"},{"name":"testing_utils.py","path":"src/transformers/testing_utils.py","contentType":"file"},{"name":"tf_utils.py","path":"src/transformers/tf_utils.py","contentType":"file"},{"name":"time_series_utils.py","path":"src/transformers/time_series_utils.py","contentType":"file"},{"name":"tokenization_utils.py","path":"src/transformers/tokenization_utils.py","contentType":"file"},{"name":"tokenization_utils_base.py","path":"src/transformers/tokenization_utils_base.py","contentType":"file"},{"name":"tokenization_utils_fast.py","path":"src/transformers/tokenization_utils_fast.py","contentType":"file"},{"name":"trainer.py","path":"src/transformers/trainer.py","contentType":"file"},{"name":"trainer_callback.py","path":"src/transformers/trainer_callback.py","contentType":"file"},{"name":"trainer_pt_utils.py","path":"src/transformers/trainer_pt_utils.py","contentType":"file"},{"name":"trainer_seq2seq.py","path":"src/transformers/trainer_seq2seq.py","contentType":"file"},{"name":"trainer_utils.py","path":"src/transformers/trainer_utils.py","contentType":"file"},{"name":"training_args.py","path":"src/transformers/training_args.py","contentType":"file"},{"name":"training_args_seq2seq.py","path":"src/transformers/training_args_seq2seq.py","contentType":"file"},{"name":"training_args_tf.py","path":"src/transformers/training_args_tf.py","contentType":"file"}],"totalCount":70},"src":{"items":[{"name":"transformers","path":"src/transformers","contentType":"directory"}],"totalCount":1},"":{"items":[{"name":".circleci","path":".circleci","contentType":"directory"},{"name":".github","path":".github","contentType":"directory"},{"name":"docker","path":"docker","contentType":"directory"},{"name":"docs","path":"docs","contentType":"directory"},{"name":"examples","path":"examples","contentType":"directory"},{"name":"model_cards","path":"model_cards","contentType":"directory"},{"name":"notebooks","path":"notebooks","contentType":"directory"},{"name":"scripts","path":"scripts","contentType":"directory"},{"name":"src","path":"src","contentType":"directory"},{"name":"templates","path":"templates","contentType":"directory"},{"name":"tests","path":"tests","contentType":"directory"},{"name":"utils","path":"utils","contentType":"directory"},{"name":".coveragerc","path":".coveragerc","contentType":"file"},{"name":".gitattributes","path":".gitattributes","contentType":"file"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":"CITATION.cff","path":"CITATION.cff","contentType":"file"},{"name":"CODE_OF_CONDUCT.md","path":"CODE_OF_CONDUCT.md","contentType":"file"},{"name":"CONTRIBUTING.md","path":"CONTRIBUTING.md","contentType":"file"},{"name":"ISSUES.md","path":"ISSUES.md","contentType":"file"},{"name":"LICENSE","path":"LICENSE","contentType":"file"},{"name":"Makefile","path":"Makefile","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"},{"name":"README_de.md","path":"README_de.md","contentType":"file"},{"name":"README_es.md","path":"README_es.md","contentType":"file"},{"name":"README_fr.md","path":"README_fr.md","contentType":"file"},{"name":"README_hd.md","path":"README_hd.md","contentType":"file"},{"name":"README_ja.md","path":"README_ja.md","contentType":"file"},{"name":"README_ko.md","path":"README_ko.md","contentType":"file"},{"name":"README_pt-br.md","path":"README_pt-br.md","contentType":"file"},{"name":"README_ru.md","path":"README_ru.md","contentType":"file"},{"name":"README_te.md","path":"README_te.md","contentType":"file"},{"name":"README_vi.md","path":"README_vi.md","contentType":"file"},{"name":"README_zh-hans.md","path":"README_zh-hans.md","contentType":"file"},{"name":"README_zh-hant.md","path":"README_zh-hant.md","contentType":"file"},{"name":"SECURITY.md","path":"SECURITY.md","contentType":"file"},{"name":"awesome-transformers.md","path":"awesome-transformers.md","contentType":"file"},{"name":"conftest.py","path":"conftest.py","contentType":"file"},{"name":"hubconf.py","path":"hubconf.py","contentType":"file"},{"name":"pyproject.toml","path":"pyproject.toml","contentType":"file"},{"name":"setup.py","path":"setup.py","contentType":"file"}],"totalCount":40}},"fileTreeProcessingTime":35.689130999999996,"foldersToFetch":[],"repo":{"id":155220641,"defaultBranch":"main","name":"transformers","ownerLogin":"huggingface","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2018-10-29T13:56:00.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/25720743?v=4","public":true,"private":false,"isOrgOwned":true},"symbolsExpanded":false,"treeExpanded":true,"refInfo":{"name":"main","listCacheKey":"v0:1710774041.0","canEdit":false,"refType":"branch","currentOid":"838b87abe231fd70be5132088d0dee72a7bb8d62"},"path":"src/transformers/models/mbart50/tokenization_mbart50.py","currentUser":null,"blob":{"rawLines":["# coding=utf-8","# Copyright 2021 The Facebook AI Research Team Authors and The HuggingFace Inc. team.","#","# Licensed under the Apache License, Version 2.0 (the \"License\");","# you may not use this file except in compliance with the License.","# You may obtain a copy of the License at","#","# http://www.apache.org/licenses/LICENSE-2.0","#","# Unless required by applicable law or agreed to in writing, software","# distributed under the License is distributed on an \"AS IS\" BASIS,","# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.","# See the License for the specific language governing permissions and","# limitations under the License.","","import os","from shutil import copyfile","from typing import Any, Dict, List, Optional, Tuple","","import sentencepiece as spm","","from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer","from ...utils import logging","","","logger = logging.get_logger(__name__)","","SPIECE_UNDERLINE = \"▁\"","","VOCAB_FILES_NAMES = {\"vocab_file\": \"sentencepiece.bpe.model\"}","","PRETRAINED_VOCAB_FILES_MAP = {"," \"vocab_file\": {"," \"facebook/mbart-large-50-one-to-many-mmt\": ("," \"https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model\""," ),"," }","}","","PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"," \"facebook/mbart-large-50-one-to-many-mmt\": 1024,","}","","FAIRSEQ_LANGUAGE_CODES = [\"ar_AR\", \"cs_CZ\", \"de_DE\", \"en_XX\", \"es_XX\", \"et_EE\", \"fi_FI\", \"fr_XX\", \"gu_IN\", \"hi_IN\", \"it_IT\", \"ja_XX\", \"kk_KZ\", \"ko_KR\", \"lt_LT\", \"lv_LV\", \"my_MM\", \"ne_NP\", \"nl_XX\", \"ro_RO\", \"ru_RU\", \"si_LK\", \"tr_TR\", \"vi_VN\", \"zh_CN\", \"af_ZA\", \"az_AZ\", \"bn_IN\", \"fa_IR\", \"he_IL\", \"hr_HR\", \"id_ID\", \"ka_GE\", \"km_KH\", \"mk_MK\", \"ml_IN\", \"mn_MN\", \"mr_IN\", \"pl_PL\", \"ps_AF\", \"pt_XX\", \"sv_SE\", \"sw_KE\", \"ta_IN\", \"te_IN\", \"th_TH\", \"tl_XX\", \"uk_UA\", \"ur_PK\", \"xh_ZA\", \"gl_ES\", \"sl_SI\"] # fmt: skip","","","class MBart50Tokenizer(PreTrainedTokenizer):"," \"\"\""," Construct a MBart50 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).",""," This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to"," this superclass for more information regarding those methods.",""," Args:"," vocab_file (`str`):"," Path to the vocabulary file."," src_lang (`str`, *optional*):"," A string representing the source language."," tgt_lang (`str`, *optional*):"," A string representing the target language."," eos_token (`str`, *optional*, defaults to `\"\"`):"," The end of sequence token."," sep_token (`str`, *optional*, defaults to `\"\"`):"," The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for"," sequence classification or for a text and a question for question answering. It is also used as the last"," token of a sequence built with special tokens."," cls_token (`str`, *optional*, defaults to `\"\"`):"," The classifier token which is used when doing sequence classification (classification of the whole sequence"," instead of per-token classification). It is the first token of the sequence when built with special tokens."," unk_token (`str`, *optional*, defaults to `\"\"`):"," The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this"," token instead."," pad_token (`str`, *optional*, defaults to `\"\"`):"," The token used for padding, for example when batching sequences of different lengths."," mask_token (`str`, *optional*, defaults to `\"\"`):"," The token used for masking values. This is the token used when training this model with masked language"," modeling. This is the token which the model will try to predict."," sp_model_kwargs (`dict`, *optional*):"," Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for"," SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,"," to set:",""," - `enable_sampling`: Enable subword regularization."," - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.",""," - `nbest_size = {0,1}`: No sampling is performed."," - `nbest_size > 1`: samples from the nbest_size results."," - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)"," using forward-filtering-and-backward-sampling algorithm.",""," - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for"," BPE-dropout.",""," Examples:",""," ```python"," >>> from transformers import MBart50Tokenizer",""," >>> tokenizer = MBart50Tokenizer.from_pretrained(\"facebook/mbart-large-50\", src_lang=\"en_XX\", tgt_lang=\"ro_RO\")"," >>> src_text = \" UN Chief Says There Is No Military Solution in Syria\""," >>> tgt_text = \"Şeful ONU declară că nu există o soluţie militară în Siria\""," >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors=\"pt\")"," >>> # model(**model_inputs) should work"," ```\"\"\"",""," vocab_files_names = VOCAB_FILES_NAMES"," max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES"," pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP"," model_input_names = [\"input_ids\", \"attention_mask\"]",""," prefix_tokens: List[int] = []"," suffix_tokens: List[int] = []",""," def __init__("," self,"," vocab_file,"," src_lang=None,"," tgt_lang=None,"," eos_token=\"\","," sep_token=\"\","," cls_token=\"\","," unk_token=\"\","," pad_token=\"\","," mask_token=\"\","," sp_model_kwargs: Optional[Dict[str, Any]] = None,"," **kwargs,"," ) -> None:"," # Mask token behave like a normal word, i.e. include the space before it"," mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token",""," self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs",""," kwargs[\"additional_special_tokens\"] = kwargs.get(\"additional_special_tokens\", []) or []"," kwargs[\"additional_special_tokens\"] += ["," code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs[\"additional_special_tokens\"]"," ]",""," self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)"," self.sp_model.Load(str(vocab_file))"," self.vocab_file = vocab_file",""," # Original fairseq vocab and spm vocab must be \"aligned\":"," # Vocab | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9"," # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----"," # fairseq | '' | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-'"," # spm | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-' | '▁a'",""," # Mimic fairseq token-to-id alignment for the first 4 token"," self.fairseq_tokens_to_ids = {\"\": 0, \"\": 1, \"\": 2, \"\": 3}",""," # The first \"real\" token \",\" has position 4 in the original fairseq vocab and position 3 in the spm vocab"," self.fairseq_offset = 1",""," self.sp_model_size = len(self.sp_model)"," self.lang_code_to_id = {"," code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(FAIRSEQ_LANGUAGE_CODES)"," }"," self.id_to_lang_code = {v: k for k, v in self.lang_code_to_id.items()}"," self.fairseq_tokens_to_ids[\"\"] = len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset",""," self.fairseq_tokens_to_ids.update(self.lang_code_to_id)"," self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}",""," super().__init__("," src_lang=src_lang,"," tgt_lang=tgt_lang,"," eos_token=eos_token,"," unk_token=unk_token,"," sep_token=sep_token,"," cls_token=cls_token,"," pad_token=pad_token,"," mask_token=mask_token,"," sp_model_kwargs=self.sp_model_kwargs,"," **kwargs,"," )",""," self._src_lang = src_lang if src_lang is not None else \"en_XX\""," self.cur_lang_code_id = self.lang_code_to_id[self._src_lang]"," self.tgt_lang = tgt_lang"," self.set_src_lang_special_tokens(self._src_lang)",""," @property"," def vocab_size(self) -> int:"," return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1 # Plus 1 for the mask token",""," @property"," def src_lang(self) -> str:"," return self._src_lang",""," @src_lang.setter"," def src_lang(self, new_src_lang: str) -> None:"," self._src_lang = new_src_lang"," self.set_src_lang_special_tokens(self._src_lang)",""," def __getstate__(self) -> Dict:"," state = self.__dict__.copy()"," state[\"sp_model\"] = None"," return state",""," def __setstate__(self, d: Dict) -> None:"," self.__dict__ = d",""," # for backward compatibility"," if not hasattr(self, \"sp_model_kwargs\"):"," self.sp_model_kwargs = {}",""," self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)"," self.sp_model.Load(self.vocab_file)",""," def get_vocab(self) -> Dict:"," vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}"," vocab.update(self.added_tokens_encoder)"," return vocab",""," def _tokenize(self, text: str) -> List[str]:"," return self.sp_model.encode(text, out_type=str)",""," def _convert_token_to_id(self, token: str) -> int:"," \"\"\"Converts a token (str) in an id using the vocab.\"\"\""," if token in self.fairseq_tokens_to_ids:"," return self.fairseq_tokens_to_ids[token]"," spm_id = self.sp_model.PieceToId(token)",""," # Need to return unknown token if the SP model returned 0"," return spm_id + self.fairseq_offset if spm_id else self.unk_token_id",""," def _convert_id_to_token(self, index: int) -> str:"," \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\""," if index in self.fairseq_ids_to_tokens:"," return self.fairseq_ids_to_tokens[index]"," return self.sp_model.IdToPiece(index - self.fairseq_offset)",""," # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.convert_tokens_to_string"," def convert_tokens_to_string(self, tokens):"," \"\"\"Converts a sequence of tokens (string) in a single string.\"\"\""," current_sub_tokens = []"," out_string = \"\""," prev_is_special = False"," for token in tokens:"," # make sure that special tokens are not decoded using sentencepiece model"," if token in self.all_special_tokens:"," if not prev_is_special:"," out_string += \" \""," out_string += self.sp_model.decode(current_sub_tokens) + token"," prev_is_special = True"," current_sub_tokens = []"," else:"," current_sub_tokens.append(token)"," prev_is_special = False"," out_string += self.sp_model.decode(current_sub_tokens)"," return out_string.strip()",""," def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:"," if not os.path.isdir(save_directory):"," logger.error(f\"Vocabulary path ({save_directory}) should be a directory\")"," return"," out_vocab_file = os.path.join("," save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"vocab_file\"]"," )",""," if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):"," copyfile(self.vocab_file, out_vocab_file)"," elif not os.path.isfile(self.vocab_file):"," with open(out_vocab_file, \"wb\") as fi:"," content_spiece_model = self.sp_model.serialized_model_proto()"," fi.write(content_spiece_model)",""," return (out_vocab_file,)",""," def get_special_tokens_mask("," self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False"," ) -> List[int]:"," \"\"\""," Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding"," special tokens using the tokenizer `prepare_for_model` method.",""," Args:"," token_ids_0 (`List[int]`):"," List of IDs."," token_ids_1 (`List[int]`, *optional*):"," Optional second list of IDs for sequence pairs."," already_has_special_tokens (`bool`, *optional*, defaults to `False`):"," Whether or not the token list is already formatted with special tokens for the model.",""," Returns:"," `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token."," \"\"\"",""," if already_has_special_tokens:"," return super().get_special_tokens_mask("," token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True"," )",""," prefix_ones = [1] * len(self.prefix_tokens)"," suffix_ones = [1] * len(self.suffix_tokens)"," if token_ids_1 is None:"," return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones"," return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones",""," def build_inputs_with_special_tokens("," self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None"," ) -> List[int]:"," \"\"\""," Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and"," adding special tokens. An MBART-50 sequence has the following format, where `X` represents the sequence:",""," - `input_ids` (for encoder) `[src_lang_code] X [eos]`"," - `labels`: (for decoder) `[tgt_lang_code] X [eos]`",""," BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a"," separator.",""," Args:"," token_ids_0 (`List[int]`):"," List of IDs to which the special tokens will be added."," token_ids_1 (`List[int]`, *optional*):"," Optional second list of IDs for sequence pairs.",""," Returns:"," `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens."," \"\"\""," if token_ids_1 is None:"," return self.prefix_tokens + token_ids_0 + self.suffix_tokens"," # We don't expect to process pairs, but leave the pair logic for API consistency"," return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens",""," def _build_translation_inputs("," self, raw_inputs, return_tensors: str, src_lang: Optional[str], tgt_lang: Optional[str], **extra_kwargs"," ):"," \"\"\"Used by translation pipeline, to prepare inputs for the generate function\"\"\""," if src_lang is None or tgt_lang is None:"," raise ValueError(\"Translation requires a `src_lang` and a `tgt_lang` for this model\")"," self.src_lang = src_lang"," inputs = self(raw_inputs, add_special_tokens=True, return_tensors=return_tensors, **extra_kwargs)"," tgt_lang_id = self.convert_tokens_to_ids(tgt_lang)"," inputs[\"forced_bos_token_id\"] = tgt_lang_id"," return inputs",""," def prepare_seq2seq_batch("," self,"," src_texts: List[str],"," src_lang: str = \"en_XX\","," tgt_texts: Optional[List[str]] = None,"," tgt_lang: str = \"ro_RO\","," **kwargs,"," ) -> BatchEncoding:"," self.src_lang = src_lang"," self.tgt_lang = tgt_lang"," return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)",""," def _switch_to_input_mode(self):"," return self.set_src_lang_special_tokens(self.src_lang)",""," def _switch_to_target_mode(self):"," return self.set_tgt_lang_special_tokens(self.tgt_lang)",""," def set_src_lang_special_tokens(self, src_lang: str) -> None:"," \"\"\"Reset the special tokens to the source lang setting. prefix=[src_lang_code] and suffix=[eos].\"\"\""," self.cur_lang_code_id = self.lang_code_to_id[src_lang]"," self.prefix_tokens = [self.cur_lang_code_id]"," self.suffix_tokens = [self.eos_token_id]",""," def set_tgt_lang_special_tokens(self, tgt_lang: str) -> None:"," \"\"\"Reset the special tokens to the target language setting. prefix=[tgt_lang_code] and suffix=[eos].\"\"\""," self.cur_lang_code_id = self.lang_code_to_id[tgt_lang]"," self.prefix_tokens = [self.cur_lang_code_id]"," self.suffix_tokens = [self.eos_token_id]"],"stylingDirectives":[[{"start":0,"end":14,"cssClass":"pl-c"}],[{"start":0,"end":85,"cssClass":"pl-c"}],[{"start":0,"end":1,"cssClass":"pl-c"}],[{"start":0,"end":65,"cssClass":"pl-c"}],[{"start":0,"end":66,"cssClass":"pl-c"}],[{"start":0,"end":41,"cssClass":"pl-c"}],[{"start":0,"end":1,"cssClass":"pl-c"}],[{"start":0,"end":48,"cssClass":"pl-c"}],[{"start":0,"end":1,"cssClass":"pl-c"}],[{"start":0,"end":69,"cssClass":"pl-c"}],[{"start":0,"end":67,"cssClass":"pl-c"}],[{"start":0,"end":74,"cssClass":"pl-c"}],[{"start":0,"end":69,"cssClass":"pl-c"}],[{"start":0,"end":32,"cssClass":"pl-c"}],[],[{"start":0,"end":6,"cssClass":"pl-k"},{"start":7,"end":9,"cssClass":"pl-s1"}],[{"start":0,"end":4,"cssClass":"pl-k"},{"start":5,"end":11,"cssClass":"pl-s1"},{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":27,"cssClass":"pl-s1"}],[{"start":0,"end":4,"cssClass":"pl-k"},{"start":5,"end":11,"cssClass":"pl-s1"},{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":22,"cssClass":"pl-v"},{"start":24,"end":28,"cssClass":"pl-v"},{"start":30,"end":34,"cssClass":"pl-v"},{"start":36,"end":44,"cssClass":"pl-v"},{"start":46,"end":51,"cssClass":"pl-v"}],[],[{"start":0,"end":6,"cssClass":"pl-k"},{"start":7,"end":20,"cssClass":"pl-s1"},{"start":21,"end":23,"cssClass":"pl-k"},{"start":24,"end":27,"cssClass":"pl-s1"}],[],[{"start":0,"end":4,"cssClass":"pl-k"},{"start":8,"end":26,"cssClass":"pl-s1"},{"start":27,"end":33,"cssClass":"pl-k"},{"start":34,"end":44,"cssClass":"pl-v"},{"start":46,"end":59,"cssClass":"pl-v"},{"start":61,"end":80,"cssClass":"pl-v"}],[{"start":0,"end":4,"cssClass":"pl-k"},{"start":8,"end":13,"cssClass":"pl-s1"},{"start":14,"end":20,"cssClass":"pl-k"},{"start":21,"end":28,"cssClass":"pl-s1"}],[],[],[{"start":0,"end":6,"cssClass":"pl-s1"},{"start":7,"end":8,"cssClass":"pl-c1"},{"start":9,"end":16,"cssClass":"pl-s1"},{"start":17,"end":27,"cssClass":"pl-en"},{"start":28,"end":36,"cssClass":"pl-s1"}],[],[{"start":0,"end":16,"cssClass":"pl-v"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":19,"end":22,"cssClass":"pl-s"}],[],[{"start":0,"end":17,"cssClass":"pl-v"},{"start":18,"end":19,"cssClass":"pl-c1"},{"start":21,"end":33,"cssClass":"pl-s"},{"start":35,"end":60,"cssClass":"pl-s"}],[],[{"start":0,"end":26,"cssClass":"pl-v"},{"start":27,"end":28,"cssClass":"pl-c1"}],[{"start":4,"end":16,"cssClass":"pl-s"}],[{"start":8,"end":49,"cssClass":"pl-s"}],[{"start":12,"end":113,"cssClass":"pl-s"}],[],[],[],[],[{"start":0,"end":38,"cssClass":"pl-v"},{"start":39,"end":40,"cssClass":"pl-c1"}],[{"start":4,"end":45,"cssClass":"pl-s"},{"start":47,"end":51,"cssClass":"pl-c1"}],[],[],[{"start":0,"end":22,"cssClass":"pl-v"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":26,"end":33,"cssClass":"pl-s"},{"start":35,"end":42,"cssClass":"pl-s"},{"start":44,"end":51,"cssClass":"pl-s"},{"start":53,"end":60,"cssClass":"pl-s"},{"start":62,"end":69,"cssClass":"pl-s"},{"start":71,"end":78,"cssClass":"pl-s"},{"start":80,"end":87,"cssClass":"pl-s"},{"start":89,"end":96,"cssClass":"pl-s"},{"start":98,"end":105,"cssClass":"pl-s"},{"start":107,"end":114,"cssClass":"pl-s"},{"start":116,"end":123,"cssClass":"pl-s"},{"start":125,"end":132,"cssClass":"pl-s"},{"start":134,"end":141,"cssClass":"pl-s"},{"start":143,"end":150,"cssClass":"pl-s"},{"start":152,"end":159,"cssClass":"pl-s"},{"start":161,"end":168,"cssClass":"pl-s"},{"start":170,"end":177,"cssClass":"pl-s"},{"start":179,"end":186,"cssClass":"pl-s"},{"start":188,"end":195,"cssClass":"pl-s"},{"start":197,"end":204,"cssClass":"pl-s"},{"start":206,"end":213,"cssClass":"pl-s"},{"start":215,"end":222,"cssClass":"pl-s"},{"start":224,"end":231,"cssClass":"pl-s"},{"start":233,"end":240,"cssClass":"pl-s"},{"start":242,"end":249,"cssClass":"pl-s"},{"start":251,"end":258,"cssClass":"pl-s"},{"start":260,"end":267,"cssClass":"pl-s"},{"start":269,"end":276,"cssClass":"pl-s"},{"start":278,"end":285,"cssClass":"pl-s"},{"start":287,"end":294,"cssClass":"pl-s"},{"start":296,"end":303,"cssClass":"pl-s"},{"start":305,"end":312,"cssClass":"pl-s"},{"start":314,"end":321,"cssClass":"pl-s"},{"start":323,"end":330,"cssClass":"pl-s"},{"start":332,"end":339,"cssClass":"pl-s"},{"start":341,"end":348,"cssClass":"pl-s"},{"start":350,"end":357,"cssClass":"pl-s"},{"start":359,"end":366,"cssClass":"pl-s"},{"start":368,"end":375,"cssClass":"pl-s"},{"start":377,"end":384,"cssClass":"pl-s"},{"start":386,"end":393,"cssClass":"pl-s"},{"start":395,"end":402,"cssClass":"pl-s"},{"start":404,"end":411,"cssClass":"pl-s"},{"start":413,"end":420,"cssClass":"pl-s"},{"start":422,"end":429,"cssClass":"pl-s"},{"start":431,"end":438,"cssClass":"pl-s"},{"start":440,"end":447,"cssClass":"pl-s"},{"start":449,"end":456,"cssClass":"pl-s"},{"start":458,"end":465,"cssClass":"pl-s"},{"start":467,"end":474,"cssClass":"pl-s"},{"start":476,"end":483,"cssClass":"pl-s"},{"start":485,"end":492,"cssClass":"pl-s"},{"start":495,"end":506,"cssClass":"pl-c"}],[],[],[{"start":0,"end":5,"cssClass":"pl-k"},{"start":6,"end":22,"cssClass":"pl-v"},{"start":23,"end":42,"cssClass":"pl-v"}],[{"start":4,"end":7,"cssClass":"pl-s"}],[{"start":0,"end":101,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":65,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":9,"cssClass":"pl-s"}],[{"start":0,"end":27,"cssClass":"pl-s"}],[{"start":0,"end":40,"cssClass":"pl-s"}],[{"start":0,"end":37,"cssClass":"pl-s"}],[{"start":0,"end":54,"cssClass":"pl-s"}],[{"start":0,"end":37,"cssClass":"pl-s"}],[{"start":0,"end":54,"cssClass":"pl-s"}],[{"start":0,"end":60,"cssClass":"pl-s"}],[{"start":0,"end":38,"cssClass":"pl-s"}],[{"start":0,"end":60,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":116,"cssClass":"pl-s"}],[{"start":0,"end":58,"cssClass":"pl-s"}],[{"start":0,"end":59,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":61,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":26,"cssClass":"pl-s"}],[{"start":0,"end":61,"cssClass":"pl-s"}],[{"start":0,"end":97,"cssClass":"pl-s"}],[{"start":0,"end":63,"cssClass":"pl-s"}],[{"start":0,"end":115,"cssClass":"pl-s"}],[{"start":0,"end":76,"cssClass":"pl-s"}],[{"start":0,"end":45,"cssClass":"pl-s"}],[{"start":0,"end":101,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":19,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":63,"cssClass":"pl-s"}],[{"start":0,"end":85,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":63,"cssClass":"pl-s"}],[{"start":0,"end":70,"cssClass":"pl-s"}],[{"start":0,"end":116,"cssClass":"pl-s"}],[{"start":0,"end":72,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":112,"cssClass":"pl-s"}],[{"start":0,"end":26,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":13,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":13,"cssClass":"pl-s"}],[{"start":0,"end":49,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":115,"cssClass":"pl-s"}],[{"start":0,"end":74,"cssClass":"pl-s"}],[{"start":0,"end":79,"cssClass":"pl-s"}],[{"start":0,"end":85,"cssClass":"pl-s"}],[{"start":0,"end":43,"cssClass":"pl-s"}],[{"start":0,"end":10,"cssClass":"pl-s"}],[],[{"start":4,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":41,"cssClass":"pl-v"}],[{"start":4,"end":25,"cssClass":"pl-s1"},{"start":26,"end":27,"cssClass":"pl-c1"},{"start":28,"end":66,"cssClass":"pl-v"}],[{"start":4,"end":30,"cssClass":"pl-s1"},{"start":31,"end":32,"cssClass":"pl-c1"},{"start":33,"end":59,"cssClass":"pl-v"}],[{"start":4,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":25,"end":36,"cssClass":"pl-s"},{"start":38,"end":54,"cssClass":"pl-s"}],[],[{"start":4,"end":17,"cssClass":"pl-s1"},{"start":19,"end":23,"cssClass":"pl-v"},{"start":24,"end":27,"cssClass":"pl-s1"},{"start":29,"end":30,"cssClass":"pl-c1"}],[{"start":4,"end":17,"cssClass":"pl-s1"},{"start":19,"end":23,"cssClass":"pl-v"},{"start":24,"end":27,"cssClass":"pl-s1"},{"start":29,"end":30,"cssClass":"pl-c1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":16,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"}],[{"start":8,"end":18,"cssClass":"pl-s1"}],[{"start":8,"end":16,"cssClass":"pl-s1"},{"start":16,"end":17,"cssClass":"pl-c1"},{"start":17,"end":21,"cssClass":"pl-c1"}],[{"start":8,"end":16,"cssClass":"pl-s1"},{"start":16,"end":17,"cssClass":"pl-c1"},{"start":17,"end":21,"cssClass":"pl-c1"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":18,"end":24,"cssClass":"pl-s"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":18,"end":24,"cssClass":"pl-s"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":18,"end":23,"cssClass":"pl-s"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":18,"end":25,"cssClass":"pl-s"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":17,"end":18,"cssClass":"pl-c1"},{"start":18,"end":25,"cssClass":"pl-s"}],[{"start":8,"end":18,"cssClass":"pl-s1"},{"start":18,"end":19,"cssClass":"pl-c1"},{"start":19,"end":27,"cssClass":"pl-s"}],[{"start":8,"end":23,"cssClass":"pl-s1"},{"start":25,"end":33,"cssClass":"pl-v"},{"start":34,"end":38,"cssClass":"pl-v"},{"start":39,"end":42,"cssClass":"pl-s1"},{"start":44,"end":47,"cssClass":"pl-v"},{"start":50,"end":51,"cssClass":"pl-c1"},{"start":52,"end":56,"cssClass":"pl-c1"}],[{"start":8,"end":10,"cssClass":"pl-c1"},{"start":10,"end":16,"cssClass":"pl-s1"}],[{"start":6,"end":8,"cssClass":"pl-c1"},{"start":9,"end":13,"cssClass":"pl-c1"}],[{"start":8,"end":80,"cssClass":"pl-c"}],[{"start":8,"end":18,"cssClass":"pl-s1"},{"start":19,"end":20,"cssClass":"pl-c1"},{"start":21,"end":31,"cssClass":"pl-v"},{"start":32,"end":42,"cssClass":"pl-s1"},{"start":44,"end":50,"cssClass":"pl-s1"},{"start":50,"end":51,"cssClass":"pl-c1"},{"start":51,"end":55,"cssClass":"pl-c1"},{"start":57,"end":63,"cssClass":"pl-s1"},{"start":63,"end":64,"cssClass":"pl-c1"},{"start":64,"end":69,"cssClass":"pl-c1"},{"start":71,"end":73,"cssClass":"pl-k"},{"start":74,"end":84,"cssClass":"pl-en"},{"start":85,"end":95,"cssClass":"pl-s1"},{"start":97,"end":100,"cssClass":"pl-s1"},{"start":102,"end":106,"cssClass":"pl-k"},{"start":107,"end":117,"cssClass":"pl-s1"}],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":28,"cssClass":"pl-s1"},{"start":29,"end":30,"cssClass":"pl-c1"},{"start":34,"end":36,"cssClass":"pl-k"},{"start":37,"end":52,"cssClass":"pl-s1"},{"start":53,"end":55,"cssClass":"pl-c1"},{"start":56,"end":60,"cssClass":"pl-c1"},{"start":61,"end":65,"cssClass":"pl-k"},{"start":66,"end":81,"cssClass":"pl-s1"}],[],[{"start":8,"end":14,"cssClass":"pl-s1"},{"start":15,"end":42,"cssClass":"pl-s"},{"start":44,"end":45,"cssClass":"pl-c1"},{"start":46,"end":52,"cssClass":"pl-s1"},{"start":53,"end":56,"cssClass":"pl-en"},{"start":57,"end":84,"cssClass":"pl-s"},{"start":90,"end":92,"cssClass":"pl-c1"}],[{"start":8,"end":14,"cssClass":"pl-s1"},{"start":15,"end":42,"cssClass":"pl-s"},{"start":44,"end":46,"cssClass":"pl-c1"}],[{"start":12,"end":16,"cssClass":"pl-s1"},{"start":17,"end":20,"cssClass":"pl-k"},{"start":21,"end":25,"cssClass":"pl-s1"},{"start":26,"end":28,"cssClass":"pl-c1"},{"start":29,"end":51,"cssClass":"pl-v"},{"start":52,"end":54,"cssClass":"pl-k"},{"start":55,"end":59,"cssClass":"pl-s1"},{"start":60,"end":63,"cssClass":"pl-c1"},{"start":64,"end":66,"cssClass":"pl-c1"},{"start":67,"end":73,"cssClass":"pl-s1"},{"start":74,"end":101,"cssClass":"pl-s"}],[],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":27,"cssClass":"pl-s1"},{"start":28,"end":50,"cssClass":"pl-v"},{"start":51,"end":53,"cssClass":"pl-c1"},{"start":53,"end":57,"cssClass":"pl-s1"},{"start":58,"end":73,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":26,"cssClass":"pl-v"},{"start":27,"end":30,"cssClass":"pl-en"},{"start":31,"end":41,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":23,"cssClass":"pl-s1"},{"start":24,"end":25,"cssClass":"pl-c1"},{"start":26,"end":36,"cssClass":"pl-s1"}],[],[{"start":8,"end":65,"cssClass":"pl-c"}],[{"start":8,"end":96,"cssClass":"pl-c"}],[{"start":8,"end":98,"cssClass":"pl-c"}],[{"start":8,"end":97,"cssClass":"pl-c"}],[{"start":8,"end":98,"cssClass":"pl-c"}],[],[{"start":8,"end":67,"cssClass":"pl-c"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":34,"cssClass":"pl-s1"},{"start":35,"end":36,"cssClass":"pl-c1"},{"start":38,"end":43,"cssClass":"pl-s"},{"start":45,"end":46,"cssClass":"pl-c1"},{"start":48,"end":55,"cssClass":"pl-s"},{"start":57,"end":58,"cssClass":"pl-c1"},{"start":60,"end":66,"cssClass":"pl-s"},{"start":68,"end":69,"cssClass":"pl-c1"},{"start":71,"end":78,"cssClass":"pl-s"},{"start":80,"end":81,"cssClass":"pl-c1"}],[],[{"start":8,"end":113,"cssClass":"pl-c"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":27,"cssClass":"pl-s1"},{"start":28,"end":29,"cssClass":"pl-c1"},{"start":30,"end":31,"cssClass":"pl-c1"}],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":29,"end":32,"cssClass":"pl-en"},{"start":33,"end":37,"cssClass":"pl-s1"},{"start":38,"end":46,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":28,"cssClass":"pl-s1"},{"start":29,"end":30,"cssClass":"pl-c1"}],[{"start":12,"end":16,"cssClass":"pl-s1"},{"start":18,"end":22,"cssClass":"pl-s1"},{"start":23,"end":36,"cssClass":"pl-s1"},{"start":37,"end":38,"cssClass":"pl-c1"},{"start":39,"end":40,"cssClass":"pl-s1"},{"start":41,"end":42,"cssClass":"pl-c1"},{"start":43,"end":47,"cssClass":"pl-s1"},{"start":48,"end":62,"cssClass":"pl-s1"},{"start":63,"end":66,"cssClass":"pl-k"},{"start":67,"end":68,"cssClass":"pl-s1"},{"start":70,"end":74,"cssClass":"pl-s1"},{"start":75,"end":77,"cssClass":"pl-c1"},{"start":78,"end":87,"cssClass":"pl-en"},{"start":88,"end":110,"cssClass":"pl-v"}],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":28,"cssClass":"pl-s1"},{"start":29,"end":30,"cssClass":"pl-c1"},{"start":32,"end":33,"cssClass":"pl-s1"},{"start":35,"end":36,"cssClass":"pl-s1"},{"start":37,"end":40,"cssClass":"pl-k"},{"start":41,"end":42,"cssClass":"pl-s1"},{"start":44,"end":45,"cssClass":"pl-s1"},{"start":46,"end":48,"cssClass":"pl-c1"},{"start":49,"end":53,"cssClass":"pl-s1"},{"start":54,"end":69,"cssClass":"pl-s1"},{"start":70,"end":75,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":34,"cssClass":"pl-s1"},{"start":35,"end":43,"cssClass":"pl-s"},{"start":45,"end":46,"cssClass":"pl-c1"},{"start":47,"end":50,"cssClass":"pl-en"},{"start":51,"end":55,"cssClass":"pl-s1"},{"start":56,"end":64,"cssClass":"pl-s1"},{"start":66,"end":67,"cssClass":"pl-c1"},{"start":68,"end":71,"cssClass":"pl-en"},{"start":72,"end":76,"cssClass":"pl-s1"},{"start":77,"end":92,"cssClass":"pl-s1"},{"start":94,"end":95,"cssClass":"pl-c1"},{"start":96,"end":100,"cssClass":"pl-s1"},{"start":101,"end":115,"cssClass":"pl-s1"}],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":34,"cssClass":"pl-s1"},{"start":35,"end":41,"cssClass":"pl-en"},{"start":42,"end":46,"cssClass":"pl-s1"},{"start":47,"end":62,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":34,"cssClass":"pl-s1"},{"start":35,"end":36,"cssClass":"pl-c1"},{"start":38,"end":39,"cssClass":"pl-s1"},{"start":41,"end":42,"cssClass":"pl-s1"},{"start":43,"end":46,"cssClass":"pl-k"},{"start":47,"end":48,"cssClass":"pl-s1"},{"start":50,"end":51,"cssClass":"pl-s1"},{"start":52,"end":54,"cssClass":"pl-c1"},{"start":55,"end":59,"cssClass":"pl-s1"},{"start":60,"end":81,"cssClass":"pl-s1"},{"start":82,"end":87,"cssClass":"pl-en"}],[],[{"start":8,"end":13,"cssClass":"pl-en"},{"start":16,"end":24,"cssClass":"pl-en"}],[{"start":12,"end":20,"cssClass":"pl-s1"},{"start":20,"end":21,"cssClass":"pl-c1"},{"start":21,"end":29,"cssClass":"pl-s1"}],[{"start":12,"end":20,"cssClass":"pl-s1"},{"start":20,"end":21,"cssClass":"pl-c1"},{"start":21,"end":29,"cssClass":"pl-s1"}],[{"start":12,"end":21,"cssClass":"pl-s1"},{"start":21,"end":22,"cssClass":"pl-c1"},{"start":22,"end":31,"cssClass":"pl-s1"}],[{"start":12,"end":21,"cssClass":"pl-s1"},{"start":21,"end":22,"cssClass":"pl-c1"},{"start":22,"end":31,"cssClass":"pl-s1"}],[{"start":12,"end":21,"cssClass":"pl-s1"},{"start":21,"end":22,"cssClass":"pl-c1"},{"start":22,"end":31,"cssClass":"pl-s1"}],[{"start":12,"end":21,"cssClass":"pl-s1"},{"start":21,"end":22,"cssClass":"pl-c1"},{"start":22,"end":31,"cssClass":"pl-s1"}],[{"start":12,"end":21,"cssClass":"pl-s1"},{"start":21,"end":22,"cssClass":"pl-c1"},{"start":22,"end":31,"cssClass":"pl-s1"}],[{"start":12,"end":22,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":23,"end":33,"cssClass":"pl-s1"}],[{"start":12,"end":27,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":28,"end":32,"cssClass":"pl-s1"},{"start":33,"end":48,"cssClass":"pl-s1"}],[{"start":12,"end":14,"cssClass":"pl-c1"},{"start":14,"end":20,"cssClass":"pl-s1"}],[],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":22,"cssClass":"pl-s1"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":25,"end":33,"cssClass":"pl-s1"},{"start":34,"end":36,"cssClass":"pl-k"},{"start":37,"end":45,"cssClass":"pl-s1"},{"start":46,"end":48,"cssClass":"pl-c1"},{"start":49,"end":52,"cssClass":"pl-c1"},{"start":53,"end":57,"cssClass":"pl-c1"},{"start":58,"end":62,"cssClass":"pl-k"},{"start":63,"end":70,"cssClass":"pl-s"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":29,"cssClass":"pl-s1"},{"start":30,"end":31,"cssClass":"pl-c1"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":52,"cssClass":"pl-s1"},{"start":53,"end":57,"cssClass":"pl-s1"},{"start":58,"end":67,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":32,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":40,"cssClass":"pl-en"},{"start":41,"end":45,"cssClass":"pl-s1"},{"start":46,"end":55,"cssClass":"pl-s1"}],[],[{"start":4,"end":13,"cssClass":"pl-en"},{"start":5,"end":13,"cssClass":"pl-s1"}],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":18,"cssClass":"pl-en"},{"start":19,"end":23,"cssClass":"pl-s1"},{"start":25,"end":27,"cssClass":"pl-c1"},{"start":28,"end":31,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":18,"cssClass":"pl-en"},{"start":19,"end":23,"cssClass":"pl-s1"},{"start":24,"end":32,"cssClass":"pl-s1"},{"start":34,"end":35,"cssClass":"pl-c1"},{"start":36,"end":39,"cssClass":"pl-en"},{"start":40,"end":44,"cssClass":"pl-s1"},{"start":45,"end":60,"cssClass":"pl-s1"},{"start":62,"end":63,"cssClass":"pl-c1"},{"start":64,"end":68,"cssClass":"pl-s1"},{"start":69,"end":83,"cssClass":"pl-s1"},{"start":84,"end":85,"cssClass":"pl-c1"},{"start":86,"end":87,"cssClass":"pl-c1"},{"start":89,"end":116,"cssClass":"pl-c"}],[],[{"start":4,"end":13,"cssClass":"pl-en"},{"start":5,"end":13,"cssClass":"pl-s1"}],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":16,"cssClass":"pl-en"},{"start":17,"end":21,"cssClass":"pl-s1"},{"start":23,"end":25,"cssClass":"pl-c1"},{"start":26,"end":29,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":29,"cssClass":"pl-s1"}],[],[{"start":4,"end":20,"cssClass":"pl-en"},{"start":5,"end":13,"cssClass":"pl-s1"},{"start":14,"end":20,"cssClass":"pl-s1"}],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":16,"cssClass":"pl-en"},{"start":17,"end":21,"cssClass":"pl-s1"},{"start":23,"end":35,"cssClass":"pl-s1"},{"start":37,"end":40,"cssClass":"pl-s1"},{"start":42,"end":44,"cssClass":"pl-c1"},{"start":45,"end":49,"cssClass":"pl-c1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":22,"cssClass":"pl-s1"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":25,"end":37,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":40,"cssClass":"pl-en"},{"start":41,"end":45,"cssClass":"pl-s1"},{"start":46,"end":55,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":20,"cssClass":"pl-en"},{"start":21,"end":25,"cssClass":"pl-s1"},{"start":27,"end":29,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-v"}],[{"start":8,"end":13,"cssClass":"pl-s1"},{"start":14,"end":15,"cssClass":"pl-c1"},{"start":16,"end":20,"cssClass":"pl-s1"},{"start":21,"end":29,"cssClass":"pl-s1"},{"start":30,"end":34,"cssClass":"pl-en"}],[{"start":8,"end":13,"cssClass":"pl-s1"},{"start":14,"end":24,"cssClass":"pl-s"},{"start":26,"end":27,"cssClass":"pl-c1"},{"start":28,"end":32,"cssClass":"pl-c1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":20,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":20,"cssClass":"pl-en"},{"start":21,"end":25,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-s1"},{"start":30,"end":34,"cssClass":"pl-v"},{"start":36,"end":38,"cssClass":"pl-c1"},{"start":39,"end":43,"cssClass":"pl-c1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":25,"cssClass":"pl-s1"}],[],[{"start":8,"end":36,"cssClass":"pl-c"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":14,"cssClass":"pl-c1"},{"start":15,"end":22,"cssClass":"pl-en"},{"start":23,"end":27,"cssClass":"pl-s1"},{"start":29,"end":46,"cssClass":"pl-s"}],[{"start":12,"end":16,"cssClass":"pl-s1"},{"start":17,"end":32,"cssClass":"pl-s1"},{"start":33,"end":34,"cssClass":"pl-c1"}],[],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":27,"cssClass":"pl-s1"},{"start":28,"end":50,"cssClass":"pl-v"},{"start":51,"end":53,"cssClass":"pl-c1"},{"start":53,"end":57,"cssClass":"pl-s1"},{"start":58,"end":73,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":26,"cssClass":"pl-v"},{"start":27,"end":31,"cssClass":"pl-s1"},{"start":32,"end":42,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":17,"cssClass":"pl-en"},{"start":18,"end":22,"cssClass":"pl-s1"},{"start":24,"end":26,"cssClass":"pl-c1"},{"start":27,"end":31,"cssClass":"pl-v"}],[{"start":8,"end":13,"cssClass":"pl-s1"},{"start":14,"end":15,"cssClass":"pl-c1"},{"start":17,"end":21,"cssClass":"pl-s1"},{"start":22,"end":43,"cssClass":"pl-en"},{"start":44,"end":45,"cssClass":"pl-s1"},{"start":48,"end":49,"cssClass":"pl-s1"},{"start":50,"end":53,"cssClass":"pl-k"},{"start":54,"end":55,"cssClass":"pl-s1"},{"start":56,"end":58,"cssClass":"pl-c1"},{"start":59,"end":64,"cssClass":"pl-en"},{"start":65,"end":69,"cssClass":"pl-s1"},{"start":70,"end":80,"cssClass":"pl-s1"}],[{"start":8,"end":13,"cssClass":"pl-s1"},{"start":14,"end":20,"cssClass":"pl-en"},{"start":21,"end":25,"cssClass":"pl-s1"},{"start":26,"end":46,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":20,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":17,"cssClass":"pl-en"},{"start":18,"end":22,"cssClass":"pl-s1"},{"start":24,"end":28,"cssClass":"pl-s1"},{"start":30,"end":33,"cssClass":"pl-s1"},{"start":35,"end":37,"cssClass":"pl-c1"},{"start":38,"end":42,"cssClass":"pl-v"},{"start":43,"end":46,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":28,"cssClass":"pl-s1"},{"start":29,"end":35,"cssClass":"pl-en"},{"start":36,"end":40,"cssClass":"pl-s1"},{"start":42,"end":50,"cssClass":"pl-s1"},{"start":50,"end":51,"cssClass":"pl-c1"},{"start":51,"end":54,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":28,"cssClass":"pl-en"},{"start":29,"end":33,"cssClass":"pl-s1"},{"start":35,"end":40,"cssClass":"pl-s1"},{"start":42,"end":45,"cssClass":"pl-s1"},{"start":47,"end":49,"cssClass":"pl-c1"},{"start":50,"end":53,"cssClass":"pl-s1"}],[{"start":8,"end":62,"cssClass":"pl-s"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":16,"cssClass":"pl-s1"},{"start":17,"end":19,"cssClass":"pl-c1"},{"start":20,"end":24,"cssClass":"pl-s1"},{"start":25,"end":46,"cssClass":"pl-s1"}],[{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":23,"cssClass":"pl-s1"},{"start":24,"end":45,"cssClass":"pl-s1"},{"start":46,"end":51,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-s1"},{"start":15,"end":16,"cssClass":"pl-c1"},{"start":17,"end":21,"cssClass":"pl-s1"},{"start":22,"end":30,"cssClass":"pl-s1"},{"start":31,"end":40,"cssClass":"pl-v"},{"start":41,"end":46,"cssClass":"pl-s1"}],[],[{"start":8,"end":65,"cssClass":"pl-c"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":28,"cssClass":"pl-s1"},{"start":29,"end":43,"cssClass":"pl-s1"},{"start":44,"end":46,"cssClass":"pl-k"},{"start":47,"end":53,"cssClass":"pl-s1"},{"start":54,"end":58,"cssClass":"pl-k"},{"start":59,"end":63,"cssClass":"pl-s1"},{"start":64,"end":76,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":28,"cssClass":"pl-en"},{"start":29,"end":33,"cssClass":"pl-s1"},{"start":35,"end":40,"cssClass":"pl-s1"},{"start":42,"end":45,"cssClass":"pl-s1"},{"start":47,"end":49,"cssClass":"pl-c1"},{"start":50,"end":53,"cssClass":"pl-s1"}],[{"start":8,"end":75,"cssClass":"pl-s"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":16,"cssClass":"pl-s1"},{"start":17,"end":19,"cssClass":"pl-c1"},{"start":20,"end":24,"cssClass":"pl-s1"},{"start":25,"end":46,"cssClass":"pl-s1"}],[{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":23,"cssClass":"pl-s1"},{"start":24,"end":45,"cssClass":"pl-s1"},{"start":46,"end":51,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":28,"cssClass":"pl-s1"},{"start":29,"end":38,"cssClass":"pl-v"},{"start":39,"end":44,"cssClass":"pl-s1"},{"start":45,"end":46,"cssClass":"pl-c1"},{"start":47,"end":51,"cssClass":"pl-s1"},{"start":52,"end":66,"cssClass":"pl-s1"}],[],[{"start":4,"end":105,"cssClass":"pl-c"}],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":32,"cssClass":"pl-en"},{"start":33,"end":37,"cssClass":"pl-s1"},{"start":39,"end":45,"cssClass":"pl-s1"}],[{"start":8,"end":72,"cssClass":"pl-s"}],[{"start":8,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"}],[{"start":8,"end":18,"cssClass":"pl-s1"},{"start":19,"end":20,"cssClass":"pl-c1"},{"start":21,"end":23,"cssClass":"pl-s"}],[{"start":8,"end":23,"cssClass":"pl-s1"},{"start":24,"end":25,"cssClass":"pl-c1"},{"start":26,"end":31,"cssClass":"pl-c1"}],[{"start":8,"end":11,"cssClass":"pl-k"},{"start":12,"end":17,"cssClass":"pl-s1"},{"start":18,"end":20,"cssClass":"pl-c1"},{"start":21,"end":27,"cssClass":"pl-s1"}],[{"start":12,"end":85,"cssClass":"pl-c"}],[{"start":12,"end":14,"cssClass":"pl-k"},{"start":15,"end":20,"cssClass":"pl-s1"},{"start":21,"end":23,"cssClass":"pl-c1"},{"start":24,"end":28,"cssClass":"pl-s1"},{"start":29,"end":47,"cssClass":"pl-s1"}],[{"start":16,"end":18,"cssClass":"pl-k"},{"start":19,"end":22,"cssClass":"pl-c1"},{"start":23,"end":38,"cssClass":"pl-s1"}],[{"start":20,"end":30,"cssClass":"pl-s1"},{"start":31,"end":33,"cssClass":"pl-c1"},{"start":34,"end":37,"cssClass":"pl-s"}],[{"start":16,"end":26,"cssClass":"pl-s1"},{"start":27,"end":29,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-s1"},{"start":35,"end":43,"cssClass":"pl-s1"},{"start":44,"end":50,"cssClass":"pl-en"},{"start":51,"end":69,"cssClass":"pl-s1"},{"start":71,"end":72,"cssClass":"pl-c1"},{"start":73,"end":78,"cssClass":"pl-s1"}],[{"start":16,"end":31,"cssClass":"pl-s1"},{"start":32,"end":33,"cssClass":"pl-c1"},{"start":34,"end":38,"cssClass":"pl-c1"}],[{"start":16,"end":34,"cssClass":"pl-s1"},{"start":35,"end":36,"cssClass":"pl-c1"}],[{"start":12,"end":16,"cssClass":"pl-k"}],[{"start":16,"end":34,"cssClass":"pl-s1"},{"start":35,"end":41,"cssClass":"pl-en"},{"start":42,"end":47,"cssClass":"pl-s1"}],[{"start":16,"end":31,"cssClass":"pl-s1"},{"start":32,"end":33,"cssClass":"pl-c1"},{"start":34,"end":39,"cssClass":"pl-c1"}],[{"start":8,"end":18,"cssClass":"pl-s1"},{"start":19,"end":21,"cssClass":"pl-c1"},{"start":22,"end":26,"cssClass":"pl-s1"},{"start":27,"end":35,"cssClass":"pl-s1"},{"start":36,"end":42,"cssClass":"pl-en"},{"start":43,"end":61,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":25,"cssClass":"pl-s1"},{"start":26,"end":31,"cssClass":"pl-en"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":23,"cssClass":"pl-en"},{"start":24,"end":28,"cssClass":"pl-s1"},{"start":30,"end":44,"cssClass":"pl-s1"},{"start":46,"end":49,"cssClass":"pl-s1"},{"start":51,"end":66,"cssClass":"pl-s1"},{"start":68,"end":76,"cssClass":"pl-v"},{"start":77,"end":80,"cssClass":"pl-s1"},{"start":82,"end":83,"cssClass":"pl-c1"},{"start":84,"end":88,"cssClass":"pl-c1"},{"start":90,"end":92,"cssClass":"pl-c1"},{"start":93,"end":98,"cssClass":"pl-v"},{"start":99,"end":102,"cssClass":"pl-s1"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":14,"cssClass":"pl-c1"},{"start":15,"end":17,"cssClass":"pl-s1"},{"start":18,"end":22,"cssClass":"pl-s1"},{"start":23,"end":28,"cssClass":"pl-en"},{"start":29,"end":43,"cssClass":"pl-s1"}],[{"start":12,"end":18,"cssClass":"pl-s1"},{"start":19,"end":24,"cssClass":"pl-en"},{"start":25,"end":84,"cssClass":"pl-s"},{"start":44,"end":60,"cssClass":"pl-s1"},{"start":44,"end":45,"cssClass":"pl-kos"},{"start":45,"end":59,"cssClass":"pl-s1"},{"start":59,"end":60,"cssClass":"pl-kos"}],[{"start":12,"end":18,"cssClass":"pl-k"}],[{"start":8,"end":22,"cssClass":"pl-s1"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":25,"end":27,"cssClass":"pl-s1"},{"start":28,"end":32,"cssClass":"pl-s1"},{"start":33,"end":37,"cssClass":"pl-en"}],[{"start":12,"end":26,"cssClass":"pl-s1"},{"start":29,"end":44,"cssClass":"pl-s1"},{"start":45,"end":46,"cssClass":"pl-c1"},{"start":47,"end":50,"cssClass":"pl-s"},{"start":51,"end":53,"cssClass":"pl-k"},{"start":54,"end":69,"cssClass":"pl-s1"},{"start":70,"end":74,"cssClass":"pl-k"},{"start":75,"end":77,"cssClass":"pl-s"},{"start":79,"end":80,"cssClass":"pl-c1"},{"start":81,"end":98,"cssClass":"pl-v"},{"start":99,"end":111,"cssClass":"pl-s"}],[],[],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":13,"cssClass":"pl-s1"},{"start":14,"end":18,"cssClass":"pl-s1"},{"start":19,"end":26,"cssClass":"pl-en"},{"start":27,"end":31,"cssClass":"pl-s1"},{"start":32,"end":42,"cssClass":"pl-s1"},{"start":44,"end":46,"cssClass":"pl-c1"},{"start":47,"end":49,"cssClass":"pl-s1"},{"start":50,"end":54,"cssClass":"pl-s1"},{"start":55,"end":62,"cssClass":"pl-en"},{"start":63,"end":77,"cssClass":"pl-s1"},{"start":79,"end":82,"cssClass":"pl-c1"},{"start":83,"end":85,"cssClass":"pl-s1"},{"start":86,"end":90,"cssClass":"pl-s1"},{"start":91,"end":97,"cssClass":"pl-en"},{"start":98,"end":102,"cssClass":"pl-s1"},{"start":103,"end":113,"cssClass":"pl-s1"}],[{"start":12,"end":20,"cssClass":"pl-en"},{"start":21,"end":25,"cssClass":"pl-s1"},{"start":26,"end":36,"cssClass":"pl-s1"},{"start":38,"end":52,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-k"},{"start":13,"end":16,"cssClass":"pl-c1"},{"start":17,"end":19,"cssClass":"pl-s1"},{"start":20,"end":24,"cssClass":"pl-s1"},{"start":25,"end":31,"cssClass":"pl-en"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":47,"cssClass":"pl-s1"}],[{"start":12,"end":16,"cssClass":"pl-k"},{"start":17,"end":21,"cssClass":"pl-en"},{"start":22,"end":36,"cssClass":"pl-s1"},{"start":38,"end":42,"cssClass":"pl-s"},{"start":44,"end":46,"cssClass":"pl-k"},{"start":47,"end":49,"cssClass":"pl-s1"}],[{"start":16,"end":36,"cssClass":"pl-s1"},{"start":37,"end":38,"cssClass":"pl-c1"},{"start":39,"end":43,"cssClass":"pl-s1"},{"start":44,"end":52,"cssClass":"pl-s1"},{"start":53,"end":75,"cssClass":"pl-en"}],[{"start":16,"end":18,"cssClass":"pl-s1"},{"start":19,"end":24,"cssClass":"pl-en"},{"start":25,"end":45,"cssClass":"pl-s1"}],[],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":16,"end":30,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":31,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":14,"end":25,"cssClass":"pl-s1"},{"start":27,"end":31,"cssClass":"pl-v"},{"start":32,"end":35,"cssClass":"pl-s1"},{"start":38,"end":49,"cssClass":"pl-s1"},{"start":51,"end":59,"cssClass":"pl-v"},{"start":60,"end":64,"cssClass":"pl-v"},{"start":65,"end":68,"cssClass":"pl-s1"},{"start":71,"end":72,"cssClass":"pl-c1"},{"start":73,"end":77,"cssClass":"pl-c1"},{"start":79,"end":105,"cssClass":"pl-s1"},{"start":107,"end":111,"cssClass":"pl-s1"},{"start":112,"end":113,"cssClass":"pl-c1"},{"start":114,"end":119,"cssClass":"pl-c1"}],[{"start":6,"end":8,"cssClass":"pl-c1"},{"start":9,"end":13,"cssClass":"pl-v"},{"start":14,"end":17,"cssClass":"pl-s1"}],[{"start":8,"end":11,"cssClass":"pl-s"}],[{"start":0,"end":115,"cssClass":"pl-s"}],[{"start":0,"end":70,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":13,"cssClass":"pl-s"}],[{"start":0,"end":38,"cssClass":"pl-s"}],[{"start":0,"end":28,"cssClass":"pl-s"}],[{"start":0,"end":50,"cssClass":"pl-s"}],[{"start":0,"end":63,"cssClass":"pl-s"}],[{"start":0,"end":81,"cssClass":"pl-s"}],[{"start":0,"end":101,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":16,"cssClass":"pl-s"}],[{"start":0,"end":111,"cssClass":"pl-s"}],[{"start":0,"end":11,"cssClass":"pl-s"}],[],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":37,"cssClass":"pl-s1"}],[{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":24,"cssClass":"pl-en"},{"start":27,"end":50,"cssClass":"pl-en"}],[{"start":16,"end":27,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":28,"end":39,"cssClass":"pl-s1"},{"start":41,"end":52,"cssClass":"pl-s1"},{"start":52,"end":53,"cssClass":"pl-c1"},{"start":53,"end":64,"cssClass":"pl-s1"},{"start":66,"end":92,"cssClass":"pl-s1"},{"start":92,"end":93,"cssClass":"pl-c1"},{"start":93,"end":97,"cssClass":"pl-c1"}],[],[],[{"start":8,"end":19,"cssClass":"pl-s1"},{"start":20,"end":21,"cssClass":"pl-c1"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":26,"end":27,"cssClass":"pl-c1"},{"start":28,"end":31,"cssClass":"pl-en"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":50,"cssClass":"pl-s1"}],[{"start":8,"end":19,"cssClass":"pl-s1"},{"start":20,"end":21,"cssClass":"pl-c1"},{"start":23,"end":24,"cssClass":"pl-c1"},{"start":26,"end":27,"cssClass":"pl-c1"},{"start":28,"end":31,"cssClass":"pl-en"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":50,"cssClass":"pl-s1"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":22,"cssClass":"pl-s1"},{"start":23,"end":25,"cssClass":"pl-c1"},{"start":26,"end":30,"cssClass":"pl-c1"}],[{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":30,"cssClass":"pl-s1"},{"start":31,"end":32,"cssClass":"pl-c1"},{"start":35,"end":36,"cssClass":"pl-c1"},{"start":38,"end":39,"cssClass":"pl-c1"},{"start":40,"end":43,"cssClass":"pl-en"},{"start":44,"end":55,"cssClass":"pl-s1"},{"start":58,"end":59,"cssClass":"pl-c1"},{"start":60,"end":71,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":31,"end":32,"cssClass":"pl-c1"},{"start":34,"end":35,"cssClass":"pl-c1"},{"start":36,"end":39,"cssClass":"pl-en"},{"start":40,"end":51,"cssClass":"pl-s1"},{"start":54,"end":55,"cssClass":"pl-c1"},{"start":58,"end":59,"cssClass":"pl-c1"},{"start":61,"end":62,"cssClass":"pl-c1"},{"start":63,"end":66,"cssClass":"pl-en"},{"start":67,"end":78,"cssClass":"pl-s1"},{"start":81,"end":82,"cssClass":"pl-c1"},{"start":83,"end":94,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":40,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":14,"end":25,"cssClass":"pl-s1"},{"start":27,"end":31,"cssClass":"pl-v"},{"start":32,"end":35,"cssClass":"pl-s1"},{"start":38,"end":49,"cssClass":"pl-s1"},{"start":51,"end":59,"cssClass":"pl-v"},{"start":60,"end":64,"cssClass":"pl-v"},{"start":65,"end":68,"cssClass":"pl-s1"},{"start":71,"end":72,"cssClass":"pl-c1"},{"start":73,"end":77,"cssClass":"pl-c1"}],[{"start":6,"end":8,"cssClass":"pl-c1"},{"start":9,"end":13,"cssClass":"pl-v"},{"start":14,"end":17,"cssClass":"pl-s1"}],[{"start":8,"end":11,"cssClass":"pl-s"}],[{"start":0,"end":119,"cssClass":"pl-s"}],[{"start":0,"end":112,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":61,"cssClass":"pl-s"}],[{"start":0,"end":59,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":111,"cssClass":"pl-s"}],[{"start":0,"end":18,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":13,"cssClass":"pl-s"}],[{"start":0,"end":38,"cssClass":"pl-s"}],[{"start":0,"end":70,"cssClass":"pl-s"}],[{"start":0,"end":50,"cssClass":"pl-s"}],[{"start":0,"end":63,"cssClass":"pl-s"}],[{"start":0,"end":0,"cssClass":"pl-s"}],[{"start":0,"end":16,"cssClass":"pl-s"}],[{"start":0,"end":104,"cssClass":"pl-s"}],[{"start":0,"end":11,"cssClass":"pl-s"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":22,"cssClass":"pl-s1"},{"start":23,"end":25,"cssClass":"pl-c1"},{"start":26,"end":30,"cssClass":"pl-c1"}],[{"start":12,"end":18,"cssClass":"pl-k"},{"start":19,"end":23,"cssClass":"pl-s1"},{"start":24,"end":37,"cssClass":"pl-s1"},{"start":38,"end":39,"cssClass":"pl-c1"},{"start":40,"end":51,"cssClass":"pl-s1"},{"start":52,"end":53,"cssClass":"pl-c1"},{"start":54,"end":58,"cssClass":"pl-s1"},{"start":59,"end":72,"cssClass":"pl-s1"}],[{"start":8,"end":88,"cssClass":"pl-c"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":33,"cssClass":"pl-s1"},{"start":34,"end":35,"cssClass":"pl-c1"},{"start":36,"end":47,"cssClass":"pl-s1"},{"start":48,"end":49,"cssClass":"pl-c1"},{"start":50,"end":61,"cssClass":"pl-s1"},{"start":62,"end":63,"cssClass":"pl-c1"},{"start":64,"end":68,"cssClass":"pl-s1"},{"start":69,"end":82,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":33,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":14,"end":24,"cssClass":"pl-s1"},{"start":26,"end":40,"cssClass":"pl-s1"},{"start":42,"end":45,"cssClass":"pl-s1"},{"start":47,"end":55,"cssClass":"pl-s1"},{"start":57,"end":65,"cssClass":"pl-v"},{"start":66,"end":69,"cssClass":"pl-s1"},{"start":72,"end":80,"cssClass":"pl-s1"},{"start":82,"end":90,"cssClass":"pl-v"},{"start":91,"end":94,"cssClass":"pl-s1"},{"start":97,"end":99,"cssClass":"pl-c1"},{"start":99,"end":111,"cssClass":"pl-s1"}],[],[{"start":8,"end":87,"cssClass":"pl-s"}],[{"start":8,"end":10,"cssClass":"pl-k"},{"start":11,"end":19,"cssClass":"pl-s1"},{"start":20,"end":22,"cssClass":"pl-c1"},{"start":23,"end":27,"cssClass":"pl-c1"},{"start":28,"end":30,"cssClass":"pl-c1"},{"start":31,"end":39,"cssClass":"pl-s1"},{"start":40,"end":42,"cssClass":"pl-c1"},{"start":43,"end":47,"cssClass":"pl-c1"}],[{"start":12,"end":17,"cssClass":"pl-k"},{"start":18,"end":28,"cssClass":"pl-v"},{"start":29,"end":96,"cssClass":"pl-s"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":32,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-s1"},{"start":15,"end":16,"cssClass":"pl-c1"},{"start":17,"end":21,"cssClass":"pl-en"},{"start":22,"end":32,"cssClass":"pl-s1"},{"start":34,"end":52,"cssClass":"pl-s1"},{"start":52,"end":53,"cssClass":"pl-c1"},{"start":53,"end":57,"cssClass":"pl-c1"},{"start":59,"end":73,"cssClass":"pl-s1"},{"start":73,"end":74,"cssClass":"pl-c1"},{"start":74,"end":88,"cssClass":"pl-s1"},{"start":90,"end":92,"cssClass":"pl-c1"},{"start":92,"end":104,"cssClass":"pl-s1"}],[{"start":8,"end":19,"cssClass":"pl-s1"},{"start":20,"end":21,"cssClass":"pl-c1"},{"start":22,"end":26,"cssClass":"pl-s1"},{"start":27,"end":48,"cssClass":"pl-en"},{"start":49,"end":57,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-s1"},{"start":15,"end":36,"cssClass":"pl-s"},{"start":38,"end":39,"cssClass":"pl-c1"},{"start":40,"end":51,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":21,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":29,"cssClass":"pl-en"}],[{"start":8,"end":12,"cssClass":"pl-s1"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":19,"end":23,"cssClass":"pl-v"},{"start":24,"end":27,"cssClass":"pl-s1"}],[{"start":8,"end":16,"cssClass":"pl-s1"},{"start":18,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":31,"cssClass":"pl-s"}],[{"start":8,"end":17,"cssClass":"pl-s1"},{"start":19,"end":27,"cssClass":"pl-v"},{"start":28,"end":32,"cssClass":"pl-v"},{"start":33,"end":36,"cssClass":"pl-s1"},{"start":39,"end":40,"cssClass":"pl-c1"},{"start":41,"end":45,"cssClass":"pl-c1"}],[{"start":8,"end":16,"cssClass":"pl-s1"},{"start":18,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":31,"cssClass":"pl-s"}],[{"start":8,"end":10,"cssClass":"pl-c1"},{"start":10,"end":16,"cssClass":"pl-s1"}],[{"start":6,"end":8,"cssClass":"pl-c1"},{"start":9,"end":22,"cssClass":"pl-v"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":32,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":21,"cssClass":"pl-s1"},{"start":22,"end":23,"cssClass":"pl-c1"},{"start":24,"end":32,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":20,"cssClass":"pl-en"},{"start":23,"end":44,"cssClass":"pl-en"},{"start":45,"end":54,"cssClass":"pl-s1"},{"start":56,"end":65,"cssClass":"pl-s1"},{"start":67,"end":69,"cssClass":"pl-c1"},{"start":69,"end":75,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":29,"cssClass":"pl-en"},{"start":30,"end":34,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":47,"cssClass":"pl-en"},{"start":48,"end":52,"cssClass":"pl-s1"},{"start":53,"end":61,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":30,"cssClass":"pl-en"},{"start":31,"end":35,"cssClass":"pl-s1"}],[{"start":8,"end":14,"cssClass":"pl-k"},{"start":15,"end":19,"cssClass":"pl-s1"},{"start":20,"end":47,"cssClass":"pl-en"},{"start":48,"end":52,"cssClass":"pl-s1"},{"start":53,"end":61,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":35,"cssClass":"pl-en"},{"start":36,"end":40,"cssClass":"pl-s1"},{"start":42,"end":50,"cssClass":"pl-s1"},{"start":52,"end":55,"cssClass":"pl-s1"},{"start":57,"end":59,"cssClass":"pl-c1"},{"start":60,"end":64,"cssClass":"pl-c1"}],[{"start":8,"end":107,"cssClass":"pl-s"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":29,"cssClass":"pl-s1"},{"start":30,"end":31,"cssClass":"pl-c1"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":52,"cssClass":"pl-s1"},{"start":53,"end":61,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-s1"},{"start":35,"end":51,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-s1"},{"start":35,"end":47,"cssClass":"pl-s1"}],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":35,"cssClass":"pl-en"},{"start":36,"end":40,"cssClass":"pl-s1"},{"start":42,"end":50,"cssClass":"pl-s1"},{"start":52,"end":55,"cssClass":"pl-s1"},{"start":57,"end":59,"cssClass":"pl-c1"},{"start":60,"end":64,"cssClass":"pl-c1"}],[{"start":8,"end":111,"cssClass":"pl-s"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":29,"cssClass":"pl-s1"},{"start":30,"end":31,"cssClass":"pl-c1"},{"start":32,"end":36,"cssClass":"pl-s1"},{"start":37,"end":52,"cssClass":"pl-s1"},{"start":53,"end":61,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-s1"},{"start":35,"end":51,"cssClass":"pl-s1"}],[{"start":8,"end":12,"cssClass":"pl-s1"},{"start":13,"end":26,"cssClass":"pl-s1"},{"start":27,"end":28,"cssClass":"pl-c1"},{"start":30,"end":34,"cssClass":"pl-s1"},{"start":35,"end":47,"cssClass":"pl-s1"}]],"colorizedLines":null,"csv":null,"csvError":null,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/huggingface/transformers/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"tokenization_mbart50.py","displayUrl":"https://github.com/huggingface/transformers/blob/main/src/transformers/models/mbart50/tokenization_mbart50.py?raw=true","headerInfo":{"blobSize":"16.4 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"cd4e52f","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Fhuggingface%2Ftransformers%2Fblob%2Fmain%2Fsrc%2Ftransformers%2Fmodels%2Fmbart50%2Ftokenization_mbart50.py","isCSV":false,"isRichtext":false,"toc":null,"lineInfo":{"truncatedLoc":"367","truncatedSloc":"302"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"issueTemplate":null,"discussionTemplate":null,"language":"Python","languageID":303,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/huggingface/transformers/blob/main/src/transformers/models/mbart50/tokenization_mbart50.py","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/huggingface/transformers/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/huggingface/transformers/raw/main/src/transformers/models/mbart50/tokenization_mbart50.py","renderImageOrRaw":false,"richText":null,"renderedFileInfo":null,"shortPath":null,"symbolsEnabled":true,"tabSize":8,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null,"symbols":{"timed_out":false,"not_analyzed":false,"symbols":[{"name":"logger","kind":"constant","ident_start":880,"ident_end":886,"extent_start":880,"extent_end":917,"fully_qualified_name":"logger","ident_utf16":{"start":{"line_number":25,"utf16_col":0},"end":{"line_number":25,"utf16_col":6}},"extent_utf16":{"start":{"line_number":25,"utf16_col":0},"end":{"line_number":25,"utf16_col":37}}},{"name":"SPIECE_UNDERLINE","kind":"constant","ident_start":919,"ident_end":935,"extent_start":919,"extent_end":943,"fully_qualified_name":"SPIECE_UNDERLINE","ident_utf16":{"start":{"line_number":27,"utf16_col":0},"end":{"line_number":27,"utf16_col":16}},"extent_utf16":{"start":{"line_number":27,"utf16_col":0},"end":{"line_number":27,"utf16_col":22}}},{"name":"VOCAB_FILES_NAMES","kind":"constant","ident_start":945,"ident_end":962,"extent_start":945,"extent_end":1006,"fully_qualified_name":"VOCAB_FILES_NAMES","ident_utf16":{"start":{"line_number":29,"utf16_col":0},"end":{"line_number":29,"utf16_col":17}},"extent_utf16":{"start":{"line_number":29,"utf16_col":0},"end":{"line_number":29,"utf16_col":61}}},{"name":"PRETRAINED_VOCAB_FILES_MAP","kind":"constant","ident_start":1008,"ident_end":1034,"extent_start":1008,"extent_end":1244,"fully_qualified_name":"PRETRAINED_VOCAB_FILES_MAP","ident_utf16":{"start":{"line_number":31,"utf16_col":0},"end":{"line_number":31,"utf16_col":26}},"extent_utf16":{"start":{"line_number":31,"utf16_col":0},"end":{"line_number":37,"utf16_col":1}}},{"name":"PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES","kind":"constant","ident_start":1246,"ident_end":1284,"extent_start":1246,"extent_end":1343,"fully_qualified_name":"PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES","ident_utf16":{"start":{"line_number":39,"utf16_col":0},"end":{"line_number":39,"utf16_col":38}},"extent_utf16":{"start":{"line_number":39,"utf16_col":0},"end":{"line_number":41,"utf16_col":1}}},{"name":"FAIRSEQ_LANGUAGE_CODES","kind":"constant","ident_start":1345,"ident_end":1367,"extent_start":1345,"extent_end":1838,"fully_qualified_name":"FAIRSEQ_LANGUAGE_CODES","ident_utf16":{"start":{"line_number":43,"utf16_col":0},"end":{"line_number":43,"utf16_col":22}},"extent_utf16":{"start":{"line_number":43,"utf16_col":0},"end":{"line_number":43,"utf16_col":493}}},{"name":"MBart50Tokenizer","kind":"class","ident_start":1860,"ident_end":1876,"extent_start":1854,"extent_end":16769,"fully_qualified_name":"MBart50Tokenizer","ident_utf16":{"start":{"line_number":46,"utf16_col":6},"end":{"line_number":46,"utf16_col":22}},"extent_utf16":{"start":{"line_number":46,"utf16_col":0},"end":{"line_number":366,"utf16_col":48}}},{"name":"vocab_files_names","kind":"constant","ident_start":5256,"ident_end":5273,"extent_start":5256,"extent_end":5293,"fully_qualified_name":"MBart50Tokenizer.vocab_files_names","ident_utf16":{"start":{"line_number":105,"utf16_col":4},"end":{"line_number":105,"utf16_col":21}},"extent_utf16":{"start":{"line_number":105,"utf16_col":4},"end":{"line_number":105,"utf16_col":41}}},{"name":"max_model_input_sizes","kind":"constant","ident_start":5298,"ident_end":5319,"extent_start":5298,"extent_end":5360,"fully_qualified_name":"MBart50Tokenizer.max_model_input_sizes","ident_utf16":{"start":{"line_number":106,"utf16_col":4},"end":{"line_number":106,"utf16_col":25}},"extent_utf16":{"start":{"line_number":106,"utf16_col":4},"end":{"line_number":106,"utf16_col":66}}},{"name":"pretrained_vocab_files_map","kind":"constant","ident_start":5365,"ident_end":5391,"extent_start":5365,"extent_end":5420,"fully_qualified_name":"MBart50Tokenizer.pretrained_vocab_files_map","ident_utf16":{"start":{"line_number":107,"utf16_col":4},"end":{"line_number":107,"utf16_col":30}},"extent_utf16":{"start":{"line_number":107,"utf16_col":4},"end":{"line_number":107,"utf16_col":59}}},{"name":"model_input_names","kind":"constant","ident_start":5425,"ident_end":5442,"extent_start":5425,"extent_end":5476,"fully_qualified_name":"MBart50Tokenizer.model_input_names","ident_utf16":{"start":{"line_number":108,"utf16_col":4},"end":{"line_number":108,"utf16_col":21}},"extent_utf16":{"start":{"line_number":108,"utf16_col":4},"end":{"line_number":108,"utf16_col":55}}},{"name":"prefix_tokens","kind":"constant","ident_start":5482,"ident_end":5495,"extent_start":5482,"extent_end":5511,"fully_qualified_name":"MBart50Tokenizer.prefix_tokens","ident_utf16":{"start":{"line_number":110,"utf16_col":4},"end":{"line_number":110,"utf16_col":17}},"extent_utf16":{"start":{"line_number":110,"utf16_col":4},"end":{"line_number":110,"utf16_col":33}}},{"name":"suffix_tokens","kind":"constant","ident_start":5516,"ident_end":5529,"extent_start":5516,"extent_end":5545,"fully_qualified_name":"MBart50Tokenizer.suffix_tokens","ident_utf16":{"start":{"line_number":111,"utf16_col":4},"end":{"line_number":111,"utf16_col":17}},"extent_utf16":{"start":{"line_number":111,"utf16_col":4},"end":{"line_number":111,"utf16_col":33}}},{"name":"__init__","kind":"function","ident_start":5555,"ident_end":5563,"extent_start":5551,"extent_end":8519,"fully_qualified_name":"MBart50Tokenizer.__init__","ident_utf16":{"start":{"line_number":113,"utf16_col":8},"end":{"line_number":113,"utf16_col":16}},"extent_utf16":{"start":{"line_number":113,"utf16_col":4},"end":{"line_number":179,"utf16_col":56}}},{"name":"vocab_size","kind":"function","ident_start":8543,"ident_end":8553,"extent_start":8539,"extent_end":8684,"fully_qualified_name":"MBart50Tokenizer.vocab_size","ident_utf16":{"start":{"line_number":182,"utf16_col":8},"end":{"line_number":182,"utf16_col":18}},"extent_utf16":{"start":{"line_number":182,"utf16_col":4},"end":{"line_number":183,"utf16_col":116}}},{"name":"src_lang","kind":"function","ident_start":8708,"ident_end":8716,"extent_start":8704,"extent_end":8760,"fully_qualified_name":"MBart50Tokenizer.src_lang","ident_utf16":{"start":{"line_number":186,"utf16_col":8},"end":{"line_number":186,"utf16_col":16}},"extent_utf16":{"start":{"line_number":186,"utf16_col":4},"end":{"line_number":187,"utf16_col":29}}},{"name":"src_lang","kind":"function","ident_start":8791,"ident_end":8799,"extent_start":8787,"extent_end":8928,"fully_qualified_name":"MBart50Tokenizer.src_lang","ident_utf16":{"start":{"line_number":190,"utf16_col":8},"end":{"line_number":190,"utf16_col":16}},"extent_utf16":{"start":{"line_number":190,"utf16_col":4},"end":{"line_number":192,"utf16_col":56}}},{"name":"__getstate__","kind":"function","ident_start":8938,"ident_end":8950,"extent_start":8934,"extent_end":9056,"fully_qualified_name":"MBart50Tokenizer.__getstate__","ident_utf16":{"start":{"line_number":194,"utf16_col":8},"end":{"line_number":194,"utf16_col":20}},"extent_utf16":{"start":{"line_number":194,"utf16_col":4},"end":{"line_number":197,"utf16_col":20}}},{"name":"__setstate__","kind":"function","ident_start":9066,"ident_end":9078,"extent_start":9062,"extent_end":9373,"fully_qualified_name":"MBart50Tokenizer.__setstate__","ident_utf16":{"start":{"line_number":199,"utf16_col":8},"end":{"line_number":199,"utf16_col":20}},"extent_utf16":{"start":{"line_number":199,"utf16_col":4},"end":{"line_number":207,"utf16_col":43}}},{"name":"get_vocab","kind":"function","ident_start":9383,"ident_end":9392,"extent_start":9379,"extent_end":9559,"fully_qualified_name":"MBart50Tokenizer.get_vocab","ident_utf16":{"start":{"line_number":209,"utf16_col":8},"end":{"line_number":209,"utf16_col":17}},"extent_utf16":{"start":{"line_number":209,"utf16_col":4},"end":{"line_number":212,"utf16_col":20}}},{"name":"_tokenize","kind":"function","ident_start":9569,"ident_end":9578,"extent_start":9565,"extent_end":9665,"fully_qualified_name":"MBart50Tokenizer._tokenize","ident_utf16":{"start":{"line_number":214,"utf16_col":8},"end":{"line_number":214,"utf16_col":17}},"extent_utf16":{"start":{"line_number":214,"utf16_col":4},"end":{"line_number":215,"utf16_col":55}}},{"name":"_convert_token_to_id","kind":"function","ident_start":9675,"ident_end":9695,"extent_start":9671,"extent_end":10077,"fully_qualified_name":"MBart50Tokenizer._convert_token_to_id","ident_utf16":{"start":{"line_number":217,"utf16_col":8},"end":{"line_number":217,"utf16_col":28}},"extent_utf16":{"start":{"line_number":217,"utf16_col":4},"end":{"line_number":224,"utf16_col":76}}},{"name":"_convert_id_to_token","kind":"function","ident_start":10087,"ident_end":10107,"extent_start":10083,"extent_end":10378,"fully_qualified_name":"MBart50Tokenizer._convert_id_to_token","ident_utf16":{"start":{"line_number":226,"utf16_col":8},"end":{"line_number":226,"utf16_col":28}},"extent_utf16":{"start":{"line_number":226,"utf16_col":4},"end":{"line_number":230,"utf16_col":67}}},{"name":"convert_tokens_to_string","kind":"function","ident_start":10494,"ident_end":10518,"extent_start":10490,"extent_end":11298,"fully_qualified_name":"MBart50Tokenizer.convert_tokens_to_string","ident_utf16":{"start":{"line_number":233,"utf16_col":8},"end":{"line_number":233,"utf16_col":32}},"extent_utf16":{"start":{"line_number":233,"utf16_col":4},"end":{"line_number":250,"utf16_col":33}}},{"name":"save_vocabulary","kind":"function","ident_start":11308,"ident_end":11323,"extent_start":11304,"extent_end":12148,"fully_qualified_name":"MBart50Tokenizer.save_vocabulary","ident_utf16":{"start":{"line_number":252,"utf16_col":8},"end":{"line_number":252,"utf16_col":23}},"extent_utf16":{"start":{"line_number":252,"utf16_col":4},"end":{"line_number":267,"utf16_col":32}}},{"name":"get_special_tokens_mask","kind":"function","ident_start":12158,"ident_end":12181,"extent_start":12154,"extent_end":13553,"fully_qualified_name":"MBart50Tokenizer.get_special_tokens_mask","ident_utf16":{"start":{"line_number":269,"utf16_col":8},"end":{"line_number":269,"utf16_col":31}},"extent_utf16":{"start":{"line_number":269,"utf16_col":4},"end":{"line_number":297,"utf16_col":94}}},{"name":"build_inputs_with_special_tokens","kind":"function","ident_start":13563,"ident_end":13595,"extent_start":13559,"extent_end":14846,"fully_qualified_name":"MBart50Tokenizer.build_inputs_with_special_tokens","ident_utf16":{"start":{"line_number":299,"utf16_col":8},"end":{"line_number":299,"utf16_col":40}},"extent_utf16":{"start":{"line_number":299,"utf16_col":4},"end":{"line_number":324,"utf16_col":82}}},{"name":"_build_translation_inputs","kind":"function","ident_start":14856,"ident_end":14881,"extent_start":14852,"extent_end":15508,"fully_qualified_name":"MBart50Tokenizer._build_translation_inputs","ident_utf16":{"start":{"line_number":326,"utf16_col":8},"end":{"line_number":326,"utf16_col":33}},"extent_utf16":{"start":{"line_number":326,"utf16_col":4},"end":{"line_number":336,"utf16_col":21}}},{"name":"prepare_seq2seq_batch","kind":"function","ident_start":15518,"ident_end":15539,"extent_start":15514,"extent_end":15882,"fully_qualified_name":"MBart50Tokenizer.prepare_seq2seq_batch","ident_utf16":{"start":{"line_number":338,"utf16_col":8},"end":{"line_number":338,"utf16_col":29}},"extent_utf16":{"start":{"line_number":338,"utf16_col":4},"end":{"line_number":348,"utf16_col":76}}},{"name":"_switch_to_input_mode","kind":"function","ident_start":15892,"ident_end":15913,"extent_start":15888,"extent_end":15983,"fully_qualified_name":"MBart50Tokenizer._switch_to_input_mode","ident_utf16":{"start":{"line_number":350,"utf16_col":8},"end":{"line_number":350,"utf16_col":29}},"extent_utf16":{"start":{"line_number":350,"utf16_col":4},"end":{"line_number":351,"utf16_col":62}}},{"name":"_switch_to_target_mode","kind":"function","ident_start":15993,"ident_end":16015,"extent_start":15989,"extent_end":16085,"fully_qualified_name":"MBart50Tokenizer._switch_to_target_mode","ident_utf16":{"start":{"line_number":353,"utf16_col":8},"end":{"line_number":353,"utf16_col":30}},"extent_utf16":{"start":{"line_number":353,"utf16_col":4},"end":{"line_number":354,"utf16_col":62}}},{"name":"set_src_lang_special_tokens","kind":"function","ident_start":16095,"ident_end":16122,"extent_start":16091,"extent_end":16425,"fully_qualified_name":"MBart50Tokenizer.set_src_lang_special_tokens","ident_utf16":{"start":{"line_number":356,"utf16_col":8},"end":{"line_number":356,"utf16_col":35}},"extent_utf16":{"start":{"line_number":356,"utf16_col":4},"end":{"line_number":360,"utf16_col":48}}},{"name":"set_tgt_lang_special_tokens","kind":"function","ident_start":16435,"ident_end":16462,"extent_start":16431,"extent_end":16769,"fully_qualified_name":"MBart50Tokenizer.set_tgt_lang_special_tokens","ident_utf16":{"start":{"line_number":362,"utf16_col":8},"end":{"line_number":362,"utf16_col":35}},"extent_utf16":{"start":{"line_number":362,"utf16_col":4},"end":{"line_number":366,"utf16_col":48}}}]}},"copilotInfo":null,"copilotAccessAllowed":false,"csrf_tokens":{"/huggingface/transformers/branches":{"post":"B7yZA23MFCk60fczI_9ONWtXz31v1OQ6R0Pobp900_q_MEngvL076_JT7pJXRxQNo_9vQmnfC7XUOHSmFhuffQ"},"/repos/preferences":{"post":"sgx8HZ8014SH7_KXJGQSG-bdJg7AFeHKigNrjqdNkWPqBxelv1hmpWKAZZ9b_f1CJItayY8DFQcHSxXPW2Q53g"}}},"title":"transformers/src/transformers/models/mbart50/tokenization_mbart50.py at main · huggingface/transformers"}