token: add tokenizer token with position skip

naoa · Aug 15, 2014 · 26e93c2 · 26e93c2
1 parent c5ec116
commit 26e93c2
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/include/groonga/tokenizer.h b/include/groonga/tokenizer.h
@@ -152,17 +152,19 @@ GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token
 typedef unsigned int grn_tokenizer_status;
 
 /* GRN_TOKENIZER_TOKEN_CONTINUE means that the next token is not the last one. */
-#define GRN_TOKENIZER_TOKEN_CONTINUE  (0)
+#define GRN_TOKENIZER_TOKEN_CONTINUE      (0)
 /* GRN_TOKENIZER_TOKEN_LAST means that the next token is the last one. */
-#define GRN_TOKENIZER_TOKEN_LAST      (0x01L<<0)
+#define GRN_TOKENIZER_TOKEN_LAST          (0x01L<<0)
 /* GRN_TOKENIZER_TOKEN_OVERLAP means that ... */
-#define GRN_TOKENIZER_TOKEN_OVERLAP   (0x01L<<1)
+#define GRN_TOKENIZER_TOKEN_OVERLAP       (0x01L<<1)
 /* GRN_TOKENIZER_TOKEN_UNMATURED means that ... */
-#define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2)
+#define GRN_TOKENIZER_TOKEN_UNMATURED     (0x01L<<2)
 /* GRN_TOKENIZER_TOKEN_REACH_END means that ... */
-#define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3)
+#define GRN_TOKENIZER_TOKEN_REACH_END     (0x01L<<3)
 /* GRN_TOKENIZER_TOKEN_SKIP means that the token is skipped */
-#define GRN_TOKENIZER_TOKEN_SKIP      (0x01L<<4)
+#define GRN_TOKENIZER_TOKEN_SKIP          (0x01L<<4)
+/* GRN_TOKENIZER_TOKEN_SKIP means that the token and postion is skipped */
+#define GRN_TOKENIZER_TOKEN_WITH_POSITION_SKIP (0x01L<<5)
 
 /*
  * GRN_TOKENIZER_CONTINUE and GRN_TOKENIZER_LAST are deprecated. They

diff --git a/lib/token.c b/lib/token.c
@@ -582,6 +582,8 @@ grn_token_next(grn_ctx *ctx, grn_token *token)
       if (status & GRN_TOKENIZER_TOKEN_SKIP) {
         token->pos++;
         continue;
+      } else if (status & GRN_TOKENIZER_TOKEN_WITH_POSITION_SKIP) {
+        continue;
       }
       if (token->curr_size == 0) {
         char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];