Show tokenizer and original text in empty token warning message

groonga · Oct 31, 2013 · cf90693 · cf90693
1 parent 56dbf3b
commit cf90693
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 1 deletion.
diff --git a/lib/token.c b/lib/token.c
@@ -576,7 +576,15 @@ grn_token_next(grn_ctx *ctx, grn_token *token)
         ? GRN_TOKEN_DONE : GRN_TOKEN_DOING;
       token->force_prefix = 0;
       if (token->curr_size == 0) {
-        GRN_LOG(ctx, GRN_WARN, "[token_next] ignore an empty token.");
+        char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE];
+        int tokenizer_name_length;
+        tokenizer_name_length =
+          grn_obj_name(ctx, token->tokenizer,
+                       tokenizer_name, GRN_TABLE_MAX_KEY_SIZE);
+        GRN_LOG(ctx, GRN_WARN,
+                "[token_next] ignore an empty token: <%.*s>: <%.*s>",
+                tokenizer_name_length, tokenizer_name,
+                token->orig_blen, token->orig);
         continue;
       }
       if (token->curr_size > GRN_TABLE_MAX_KEY_SIZE) {

diff --git a/test/command/suite/tokenize/empty_token.expected b/test/command/suite/tokenize/empty_token.expected
@@ -0,0 +1,3 @@
+tokenize TokenDelimit "aBc" NormalizerAuto ENABLE_TOKENIZED_DELIMITER
+[[0,0.0,0.0],[{"value":"ab","position":0},{"value":"c","position":1}]]
+#|w| [token_next] ignore an empty token: <TokenDelimit>: <aBc>
diff --git a/test/command/suite/tokenize/empty_token.test b/test/command/suite/tokenize/empty_token.test
@@ -0,0 +1 @@
+tokenize TokenDelimit "aBc" NormalizerAuto ENABLE_TOKENIZED_DELIMITER