Add snippet_full() function

It uses the full-spec on the snippet API.
naoa · Jan 20, 2014 · af6ef72 · af6ef72
1 parent 98d421a
commit af6ef72
Show file tree

Hide file tree

Showing 33 changed files with 1,226 additions and 0 deletions.
diff --git a/lib/proc.c b/lib/proc.c
@@ -3697,6 +3697,75 @@ func_snippet_html(grn_ctx *ctx, int nargs, grn_obj **args,
   return snippets;
 }
 
+static grn_obj *
+func_snippet_full(grn_ctx *ctx, int nargs, grn_obj **args,
+                  grn_user_data *user_data)
+{
+  grn_obj *snippets;
+
+  snippets = GRN_PROC_ALLOC(GRN_DB_SHORT_TEXT, GRN_OBJ_VECTOR);
+  if (!snippets) {
+    return NULL;
+  }
+
+  if (nargs > 10) {
+    grn_obj *text = args[0];
+    grn_snip *snip = NULL;
+    unsigned int width = GRN_UINT64_VALUE(args[1]);
+    unsigned int max_n_results = GRN_UINT64_VALUE(args[2]);
+    grn_snip_mapping *mapping = NULL;
+
+    int flags = GRN_SNIP_COPY_TAG;
+
+    if(GRN_TEXT_LEN(args[3])){
+      flags |= GRN_SNIP_NORMALIZE;
+      /* TODO: add use grn_snip_set_normalizer */
+    }
+    if(GRN_UINT64_VALUE(args[4])){
+      flags |= GRN_SNIP_SKIP_LEADING_SPACES;
+    }
+    if(GRN_UINT64_VALUE(args[5])){
+      mapping = GRN_SNIP_MAPPING_HTML_ESCAPE;
+    }
+    snip = grn_snip_open(ctx, flags, width, max_n_results, "", 0, "", 0, mapping);
+
+    if (snip) {
+      grn_rc rc;
+      unsigned int i, n_results, max_tagged_length;
+      grn_obj snippet_buffer;
+
+      for(i = 8; i < nargs; i += 3){
+        rc = grn_snip_add_cond(ctx, snip,
+                               GRN_TEXT_VALUE(args[i]), GRN_TEXT_LEN(args[i]),
+                               GRN_TEXT_VALUE(args[i + 1]), GRN_TEXT_LEN(args[i + 1]),
+                               GRN_TEXT_VALUE(args[i + 2]), GRN_TEXT_LEN(args[i + 2]));
+      }
+      rc = grn_snip_exec(ctx, snip,
+                         GRN_TEXT_VALUE(text), GRN_TEXT_LEN(text),
+                         &n_results, &max_tagged_length);
+      GRN_TEXT_INIT(&snippet_buffer, 0);
+      grn_bulk_space(ctx, &snippet_buffer, max_tagged_length);
+      for (i = 0; i < n_results; i++) {
+        unsigned int snippet_length;
+
+        GRN_BULK_REWIND(&snippet_buffer);
+        GRN_TEXT_PUT(ctx,&snippet_buffer,GRN_TEXT_VALUE(args[6]),GRN_TEXT_LEN(args[6]));
+        rc = grn_snip_get_result(ctx, snip, i,
+                                 GRN_TEXT_VALUE(&snippet_buffer) + GRN_TEXT_LEN(args[6]),
+                                 &snippet_length);
+        grn_vector_add_element(ctx, snippets,
+                               strcat(GRN_TEXT_VALUE(&snippet_buffer), GRN_TEXT_VALUE(args[7])),
+                               GRN_TEXT_LEN(args[6]) + snippet_length + GRN_TEXT_LEN(args[7]),
+                               0, GRN_DB_SHORT_TEXT);
+      }
+      GRN_OBJ_FIN(ctx, &snippet_buffer);
+      grn_snip_close(ctx, snip);
+    }
+  }
+
+  return snippets;
+}
+
 typedef struct {
   grn_obj *found;
   grn_obj *table;
@@ -4580,6 +4649,9 @@ grn_db_init_builtin_query(grn_ctx *ctx)
   grn_proc_create(ctx, "snippet_html", -1, GRN_PROC_FUNCTION,
                   func_snippet_html, NULL, NULL, 0, NULL);
 
+  grn_proc_create(ctx, "snippet_full", -1, GRN_PROC_FUNCTION,
+                  func_snippet_full, NULL, NULL, 0, NULL);
+
   {
     grn_obj *selector_proc;
 

diff --git a/test/command/suite/select/function/snippet_full/html_escape_off.expected b/test/command/suite/select/function/snippet_full/html_escape_off.expected
@@ -0,0 +1,53 @@
+table_create Entries TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Entries title COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Entries content COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+[[0,0.0,0.0],true]
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+[[0,0.0,0.0],true]
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+[[0,0.0,0.0],true]
+load --table Entries
+[
+{"title": "<p>groonga and MySQL</p>", "content": "groonga + MySQL = mroonga."}
+]
+[[0,0.0,0.0],1]
+select Entries   --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 0, "...", "...",   "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content'   --command_version 2   --match_columns 'title'   --query groonga
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        1
+      ],
+      [
+        [
+          "title",
+          "ShortText"
+        ],
+        [
+          "snippet_full",
+          "null"
+        ],
+        [
+          "content",
+          "ShortText"
+        ]
+      ],
+      [
+        "<p>groonga and MySQL</p>",
+        [
+          "...<p><span class=\"w1\">groonga</span> and <span class=\"w2\">MySQL</span></p>..."
+        ],
+        "groonga + MySQL = mroonga."
+      ]
+    ]
+  ]
+]
diff --git a/test/command/suite/select/function/snippet_full/html_escape_off.test b/test/command/suite/select/function/snippet_full/html_escape_off.test
@@ -0,0 +1,19 @@
+table_create Entries TABLE_NO_KEY
+column_create Entries title COLUMN_SCALAR ShortText
+column_create Entries content COLUMN_SCALAR ShortText
+
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+
+load --table Entries
+[
+{"title": "<p>groonga and MySQL</p>", "content": "groonga + MySQL = mroonga."}
+]
+
+select Entries \
+  --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 0, "...", "...", \
+  "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content' \
+  --command_version 2 \
+  --match_columns 'title' \
+  --query groonga
diff --git a/test/command/suite/select/function/snippet_full/html_escape_on.expected b/test/command/suite/select/function/snippet_full/html_escape_on.expected
@@ -0,0 +1,53 @@
+table_create Entries TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Entries title COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Entries content COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+[[0,0.0,0.0],true]
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+[[0,0.0,0.0],true]
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+[[0,0.0,0.0],true]
+load --table Entries
+[
+{"title": "<p>groonga and MySQL</p>", "content": "groonga + MySQL = mroonga."}
+]
+[[0,0.0,0.0],1]
+select Entries   --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 1, "...", "...",   "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content'   --command_version 2   --match_columns 'title'   --query groonga
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        1
+      ],
+      [
+        [
+          "title",
+          "ShortText"
+        ],
+        [
+          "snippet_full",
+          "null"
+        ],
+        [
+          "content",
+          "ShortText"
+        ]
+      ],
+      [
+        "<p>groonga and MySQL</p>",
+        [
+          "...&lt;p&gt;<span class=\"w1\">groonga</span> and <span class=\"w2\">MySQL</span>&lt;/p&gt;..."
+        ],
+        "groonga + MySQL = mroonga."
+      ]
+    ]
+  ]
+]
diff --git a/test/command/suite/select/function/snippet_full/html_escape_on.test b/test/command/suite/select/function/snippet_full/html_escape_on.test
@@ -0,0 +1,19 @@
+table_create Entries TABLE_NO_KEY
+column_create Entries title COLUMN_SCALAR ShortText
+column_create Entries content COLUMN_SCALAR ShortText
+
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+
+load --table Entries
+[
+{"title": "<p>groonga and MySQL</p>", "content": "groonga + MySQL = mroonga."}
+]
+
+select Entries \
+  --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 1, "...", "...", \
+  "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content' \
+  --command_version 2 \
+  --match_columns 'title' \
+  --query groonga
diff --git a/test/command/suite/select/function/snippet_full/leading_space_off.expected b/test/command/suite/select/function/snippet_full/leading_space_off.expected
@@ -0,0 +1,53 @@
+table_create Entries TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Entries title COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Entries content COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+[[0,0.0,0.0],true]
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+[[0,0.0,0.0],true]
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+[[0,0.0,0.0],true]
+load --table Entries
+[
+{"title": "groonga and MySQL", "content": "groonga + MySQL = mroonga."}
+]
+[[0,0.0,0.0],1]
+select Entries   --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 0, 1, "...", "...",   "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content'   --command_version 2   --match_columns 'title'   --query groonga
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        1
+      ],
+      [
+        [
+          "title",
+          "ShortText"
+        ],
+        [
+          "snippet_full",
+          "null"
+        ],
+        [
+          "content",
+          "ShortText"
+        ]
+      ],
+      [
+        "groonga and MySQL",
+        [
+          "...<span class=\"w1\">groonga</span> and<span class=\"w2\"> MySQL</span>..."
+        ],
+        "groonga + MySQL = mroonga."
+      ]
+    ]
+  ]
+]
diff --git a/test/command/suite/select/function/snippet_full/leading_space_off.test b/test/command/suite/select/function/snippet_full/leading_space_off.test
@@ -0,0 +1,19 @@
+table_create Entries TABLE_NO_KEY
+column_create Entries title COLUMN_SCALAR ShortText
+column_create Entries content COLUMN_SCALAR ShortText
+
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+
+load --table Entries
+[
+{"title": "groonga and MySQL", "content": "groonga + MySQL = mroonga."}
+]
+
+select Entries \
+  --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 0, 1, "...", "...", \
+  "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content' \
+  --command_version 2 \
+  --match_columns 'title' \
+  --query groonga
diff --git a/test/command/suite/select/function/snippet_full/leading_space_on.expected b/test/command/suite/select/function/snippet_full/leading_space_on.expected
@@ -0,0 +1,53 @@
+table_create Entries TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Entries title COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+column_create Entries content COLUMN_SCALAR ShortText
+[[0,0.0,0.0],true]
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+[[0,0.0,0.0],true]
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+[[0,0.0,0.0],true]
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+[[0,0.0,0.0],true]
+load --table Entries
+[
+{"title": "groonga and MySQL", "content": "groonga + MySQL = mroonga."}
+]
+[[0,0.0,0.0],1]
+select Entries   --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 1, "...", "...",   "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content'   --command_version 2   --match_columns 'title'   --query groonga
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  [
+    [
+      [
+        1
+      ],
+      [
+        [
+          "title",
+          "ShortText"
+        ],
+        [
+          "snippet_full",
+          "null"
+        ],
+        [
+          "content",
+          "ShortText"
+        ]
+      ],
+      [
+        "groonga and MySQL",
+        [
+          "...<span class=\"w1\">groonga</span> and <span class=\"w2\">MySQL</span>..."
+        ],
+        "groonga + MySQL = mroonga."
+      ]
+    ]
+  ]
+]
diff --git a/test/command/suite/select/function/snippet_full/leading_space_on.test b/test/command/suite/select/function/snippet_full/leading_space_on.test
@@ -0,0 +1,19 @@
+table_create Entries TABLE_NO_KEY
+column_create Entries title COLUMN_SCALAR ShortText
+column_create Entries content COLUMN_SCALAR ShortText
+
+table_create Tokens TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram
+column_create Tokens entries_title COLUMN_INDEX|WITH_POSITION Entries title
+column_create Tokens entries_content COLUMN_INDEX|WITH_POSITION Entries content
+
+load --table Entries
+[
+{"title": "groonga and MySQL", "content": "groonga + MySQL = mroonga."}
+]
+
+select Entries \
+  --output_columns 'title, snippet_full(title, 200, 2, "NormalizerAuto", 1, 1, "...", "...", \
+  "groonga", "<span class=\\"w1\\">", "</span>", "MySQL", "<span class=\\"w2\\">", "</span>"), content' \
+  --command_version 2 \
+  --match_columns 'title' \
+  --query groonga