Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixed tokenizer to emit last word

  • Loading branch information...
commit 09e991e9ef85fe55f551a5bff2eb4dd48b559d0e 1 parent e3257d7
Dmitry Lipovoi authored
Showing with 5 additions and 1 deletion.
  1. +5 −1 objc/ferret/UnicodeWhitespaceTokenizer.h
View
6 objc/ferret/UnicodeWhitespaceTokenizer.h
@@ -80,13 +80,17 @@ static FrtToken *unicode_ws_next(FrtTokenStream *ts)
break;
}
} else {
- if (!u_isblank(c) && !u_ispunct(c)) {
+ if (u_isprint(c) && !u_isblank(c) && !u_ispunct(c)) {
start = end = utext_getPreviousNativeIndex(text);
digit = u_isdigit(c);
word = true;
}
}
}
+
+ if (word && c == U_SENTINEL) {
+ end = utext_getPreviousNativeIndex(text) + 1;
+ }
if (start == end) return NULL;
Please sign in to comment.
Something went wrong with that request. Please try again.