Permalink
Fetching contributors…
Cannot retrieve contributors at this time
90 lines (68 sloc) 2.76 KB
! tokenize.xfst
!
! Released under the terms of the MIT License
! Copyright (c) 2011-2015 Çağrı Çöltekin <cagri@coltekin.net>
!
! Permission is hereby granted, free of charge, to any person obtaining a
! copy of this software and associated documentation files (the "Software"),
! to deal in the Software without restriction, including without limitation
! the rights to use, copy, modify, merge, publish, distribute, sublicense,
! and/or sell copies of the Software, and to permit persons to whom the
! Software is furnished to do so, subject to the following conditions:
!
! The above copyright notice and this permission notice shall be included in
! all copies or substantial portions of the Software.
!
! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
! AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
! LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
! FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
! DEALINGS IN THE SOFTWARE.
!
!
! This FST is a *very* simple tokenizer that splits given input to
! tokens and sentences.
! White space
define WS " "|"\u0009"|"\u000a"|"\u000d"|
"\u00a0"|"\u1680"|
"\u2000"|"\u2001"|"\u2002"|"\u2003"|"\u2004"|"\u2005"|
"\u2006"|"\u2007"|"\u2008"|"\u2009"|"\u200a"|
"\u2028"|"\u2029"|"\u202f"|"\u205f"|"\u3000";
! Punctuation that ends sentences
define SP "..."|"."|"?"|"!"|"…";
! Left punctuation
define LP "("|"["|"{"|
"“"|"‘"|"‹"|"«"|
"'"|%"|"''";
! Right punctuation - excluding the characters that can be used as apostrophe
define RP SP|","|";"|":"|
")"|"]"|"{"|
"”"|"›"|"»"|
%"|"''";
define Sym "-"|"+"|"<"|">"|"*"|"/";
define Punct LP|RP|Sym;
define Apos %'|%’;
!define TB "\u000a";
!define SB "\u000a\u000a";
define TB "<tb>";
define SB "<sb>";
define Digit [%0|1|2|3|4|5|6|7|8|9];
define define CAP [A|B|C|Ç|D|E|F|G|Ğ|H|I|İ|J|K|L|M|N|O|Ö|P|R|S|Ş|T|U|Ü|V|Y|Z|Î|Û|Â|X|W|Q];
define define LC [a|b|c|ç|d|e|f|g|ğ|h|i|i|j|k|l|m|n|o|ö|p|r|s|ş|t|u|ü|v|y|z|î|û|â|x|w|q];
define nonSym \[WS|LP|RP|Sym];
define Word nonSym+ [(["."|"-"|Apos] nonSym+)*];
define Initial \[WS|LP|RP|Sym] ".";
define Number Digit+ [([%.|%,] Digit+)*];
define Abbr @txt"abbr.txt";
define T [Word|Punct|Initial|Number] @-> ... TB
.o. Apos @-> TB ... || _ TB
.o. TB -> SB || [SP|.#.] _
.o. SB -> TB || [Abbr|Digit+] SP _
.o. SB -> TB || LP .* SP _ RP
.o. WS+ -> 0
.o. TB -> " "
.o. SB -> "\u000a";
regex T.i;
save stack tokenize.fst