token: add tokenizer token skip

naoa · Aug 14, 2014 · c5ec116 · c5ec116
1 parent 7fc8677
commit c5ec116
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/include/groonga/tokenizer.h b/include/groonga/tokenizer.h
@@ -161,6 +161,8 @@ typedef unsigned int grn_tokenizer_status;
 #define GRN_TOKENIZER_TOKEN_UNMATURED (0x01L<<2)
 /* GRN_TOKENIZER_TOKEN_REACH_END means that ... */
 #define GRN_TOKENIZER_TOKEN_REACH_END (0x01L<<3)
+/* GRN_TOKENIZER_TOKEN_SKIP means that the token is skipped */
+#define GRN_TOKENIZER_TOKEN_SKIP      (0x01L<<4)
 
 /*
  * GRN_TOKENIZER_CONTINUE and GRN_TOKENIZER_LAST are deprecated. They

diff --git a/lib/token.c b/lib/token.c
@@ -579,6 +579,10 @@ grn_token_next(grn_ctx *ctx, grn_token *token)
                         (status & GRN_TOKENIZER_TOKEN_REACH_END)))
         ? GRN_TOKEN_DONE : GRN_TOKEN_DOING;
       token->force_prefix = 0;
+      if (status & GRN_TOKENIZER_TOKEN_SKIP) {
+        token->pos++;
+        continue;
+      }
       if (token->curr_size == 0) {
         char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
         int tokenizer_name_length;