Fixed tokenization bug, which allowed whitespaces inside punctiation …

…tokens.
clulab · Feb 27, 2018 · b3aa05d · b3aa05d
1 parent bdb2b90
commit b3aa05d
Show file tree

Hide file tree

Showing 5 changed files with 231 additions and 226 deletions.
diff --git a/antlr_tokenizer b/antlr_tokenizer
@@ -5,4 +5,4 @@
 #  java -jar <full path to your antlr-4.x-complete.jar file>
 #
 
-antlr4 main/src/main/java/org/clulab/processors/clulab/tokenizer/OpenDomainLexer.g
+antlr4 main/src/main/java/org/clulab/processors/clu/tokenizer/OpenDomainLexer.g
diff --git a/main/src/main/java/org/clulab/processors/clu/tokenizer/OpenDomainLexer.g b/main/src/main/java/org/clulab/processors/clu/tokenizer/OpenDomainLexer.g
@@ -12,7 +12,7 @@ options {
 }
 
 @lexer::header {
-  package org.clulab.processors.clulab.tokenizer;
+  package org.clulab.processors.clu.tokenizer;
 }
 
 // parentheses in Treebank and OntoNotes
@@ -65,7 +65,7 @@ SMILEY: ('<'|'>')? (':'|';'|'=') ('-'|'o'|'*'|'\'')? ('('|')'|'D'|'P'|'d'|'p'|'O
 // TODO: phone numbers
 
 // punctuation
-EOS: PUNCTUATION (WHITESPACE? PUNCTUATION)* ;
+EOS: PUNCTUATION (PUNCTUATION)* ;
 
 // skip all white spaces
 WHITESPACE: ('\t'|' '|'\r'|'\n'|'\u000C'| '\u2028'|'\u2029'|'\u000B'|'\u0085'|'\u00A0'|('\u2000'..'\u200A')|'\u3000')+ -> skip ;