GitHub - fullstackenviormentss/agile_tokenizer: English, French, and German tokenizer used by the Agile team in GALE/BOLT

originally found at /home/nlg-02/data07/bin/tokenizers/eng-tok/

Name		Name	Last commit message	Last commit date
Latest commit History 3 Commits
.gitignore		.gitignore
README.md		README.md
demo.in		demo.in
demo.tok		demo.tok
gale-de-tok.sh		gale-de-tok.sh
gale-eng-tok.sh		gale-eng-tok.sh
gale-fr-tok.sh		gale-fr-tok.sh
lw_detokenize.pl		lw_detokenize.pl
lw_tokenize.pl		lw_tokenize.pl
lw_tokenize_de_ptb.conf.v2		lw_tokenize_de_ptb.conf.v2
lw_tokenize_de_ptb2mt.conf		lw_tokenize_de_ptb2mt.conf
lw_tokenize_eng_ptb.conf.v2		lw_tokenize_eng_ptb.conf.v2
lw_tokenize_eng_ptb2mt.conf		lw_tokenize_eng_ptb2mt.conf
lw_tokenize_fr_ptb.conf.v2		lw_tokenize_fr_ptb.conf.v2
lw_tokenize_fr_ptb2mt.conf		lw_tokenize_fr_ptb2mt.conf
nonbreaking_prefix.de		nonbreaking_prefix.de
usage		usage

fullstackenviormentss/agile_tokenizer