schema: support token_filters

groonga · Oct 20, 2015 · 1149af3 · 1149af3
1 parent 416cad0
commit 1149af3
Show file tree

Hide file tree

Showing 9 changed files with 265 additions and 8 deletions.
diff --git a/lib/proc.c b/lib/proc.c
@@ -7546,6 +7546,36 @@ proc_schema_table_output_normalizer(grn_ctx *ctx, grn_obj *table)
   proc_schema_output_name(ctx, normalizer);
 }
 
+static void
+proc_schema_table_output_token_filters(grn_ctx *ctx, grn_obj *table)
+{
+  grn_obj token_filters;
+  int i, n;
+
+  GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_DB_OBJECT);
+  if (table->header.type != GRN_TABLE_NO_KEY) {
+  grn_obj_get_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters);
+  }
+
+  n = GRN_BULK_VSIZE(&token_filters) / sizeof(grn_obj *);
+  GRN_OUTPUT_ARRAY_OPEN("token_filters", n);
+  for (i = 0; i < n; i++) {
+    grn_obj *token_filter;
+
+    token_filter = GRN_PTR_VALUE_AT(&token_filters, i);
+
+    GRN_OUTPUT_MAP_OPEN("token_filter", 1);
+
+    GRN_OUTPUT_CSTR("name");
+    proc_schema_output_name(ctx, token_filter);
+
+    GRN_OUTPUT_MAP_CLOSE();
+  }
+  GRN_OUTPUT_ARRAY_CLOSE();
+
+  GRN_OBJ_FIN(ctx, &token_filters);
+}
+
 static void
 proc_schema_tables(grn_ctx *ctx)
 {
@@ -7567,7 +7597,7 @@ proc_schema_tables(grn_ctx *ctx)
 
     proc_schema_output_name(ctx, table);
 
-    GRN_OUTPUT_MAP_OPEN("table", 6);
+    GRN_OUTPUT_MAP_OPEN("table", 7);
 
     GRN_OUTPUT_CSTR("name");
     proc_schema_output_name(ctx, table);
@@ -7587,6 +7617,9 @@ proc_schema_tables(grn_ctx *ctx)
     GRN_OUTPUT_CSTR("normalizer");
     proc_schema_table_output_normalizer(ctx, table);
 
+    GRN_OUTPUT_CSTR("token_filters");
+    proc_schema_table_output_token_filters(ctx, table);
+
     GRN_OUTPUT_MAP_CLOSE();
   }
   GRN_OUTPUT_MAP_CLOSE();

diff --git a/test/command/suite/schema/tables/normalizer.expected b/test/command/suite/schema/tables/normalizer.expected
@@ -176,7 +176,10 @@ schema
         },
         "value_type": null,
         "tokenizer": null,
-        "normalizer": "NormalizerAuto"
+        "normalizer": "NormalizerAuto",
+        "token_filters": [
+
+        ]
       }
     }
   }

diff --git a/test/command/suite/schema/tables/token_filters.expected b/test/command/suite/schema/tables/token_filters.expected
@@ -0,0 +1,196 @@
+plugin_register token_filters/stop_word
+[[0,0.0,0.0],true]
+table_create Terms TABLE_PAT_KEY ShortText   --default_tokenizer TokenBigram   --token_filters TokenFilterStopWord
+[[0,0.0,0.0],true]
+schema
+[
+  [
+    0,
+    0.0,
+    0.0
+  ],
+  {
+    "plugins": {
+      "token_filters/stop_word": {
+        "name": "token_filters/stop_word"
+      }
+    },
+    "types": {
+      "Bool": {
+        "name": "Bool",
+        "size": 1,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "Float": {
+        "name": "Float",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "Int16": {
+        "name": "Int16",
+        "size": 2,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "Int32": {
+        "name": "Int32",
+        "size": 4,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "Int64": {
+        "name": "Int64",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "Int8": {
+        "name": "Int8",
+        "size": 1,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "LongText": {
+        "name": "LongText",
+        "size": 2147483648,
+        "can_be_key_type": false,
+        "can_be_value_type": false
+      },
+      "Object": {
+        "name": "Object",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "ShortText": {
+        "name": "ShortText",
+        "size": 4096,
+        "can_be_key_type": true,
+        "can_be_value_type": false
+      },
+      "Text": {
+        "name": "Text",
+        "size": 65536,
+        "can_be_key_type": false,
+        "can_be_value_type": false
+      },
+      "Time": {
+        "name": "Time",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "TokyoGeoPoint": {
+        "name": "TokyoGeoPoint",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "UInt16": {
+        "name": "UInt16",
+        "size": 2,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "UInt32": {
+        "name": "UInt32",
+        "size": 4,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "UInt64": {
+        "name": "UInt64",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "UInt8": {
+        "name": "UInt8",
+        "size": 1,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      },
+      "WGS84GeoPoint": {
+        "name": "WGS84GeoPoint",
+        "size": 8,
+        "can_be_key_type": true,
+        "can_be_value_type": true
+      }
+    },
+    "tokenizers": {
+      "TokenBigram": {
+        "name": "TokenBigram"
+      },
+      "TokenBigramIgnoreBlank": {
+        "name": "TokenBigramIgnoreBlank"
+      },
+      "TokenBigramIgnoreBlankSplitSymbol": {
+        "name": "TokenBigramIgnoreBlankSplitSymbol"
+      },
+      "TokenBigramIgnoreBlankSplitSymbolAlpha": {
+        "name": "TokenBigramIgnoreBlankSplitSymbolAlpha"
+      },
+      "TokenBigramIgnoreBlankSplitSymbolAlphaDigit": {
+        "name": "TokenBigramIgnoreBlankSplitSymbolAlphaDigit"
+      },
+      "TokenBigramSplitSymbol": {
+        "name": "TokenBigramSplitSymbol"
+      },
+      "TokenBigramSplitSymbolAlpha": {
+        "name": "TokenBigramSplitSymbolAlpha"
+      },
+      "TokenBigramSplitSymbolAlphaDigit": {
+        "name": "TokenBigramSplitSymbolAlphaDigit"
+      },
+      "TokenDelimit": {
+        "name": "TokenDelimit"
+      },
+      "TokenDelimitNull": {
+        "name": "TokenDelimitNull"
+      },
+      "TokenFilterStopWord": {
+        "name": "TokenFilterStopWord"
+      },
+      "TokenMecab": {
+        "name": "TokenMecab"
+      },
+      "TokenRegexp": {
+        "name": "TokenRegexp"
+      },
+      "TokenTrigram": {
+        "name": "TokenTrigram"
+      },
+      "TokenUnigram": {
+        "name": "TokenUnigram"
+      }
+    },
+    "normalizers": {
+      "NormalizerAuto": {
+        "name": "NormalizerAuto"
+      },
+      "NormalizerNFKC51": {
+        "name": "NormalizerNFKC51"
+      }
+    },
+    "tables": {
+      "Terms": {
+        "name": "Terms",
+        "type": "patricia trie",
+        "key_type": {
+          "name": "ShortText",
+          "type": "type"
+        },
+        "value_type": null,
+        "tokenizer": "TokenBigram",
+        "normalizer": null,
+        "token_filters": [
+          {
+            "name": "TokenFilterStopWord"
+          }
+        ]
+      }
+    }
+  }
+]
diff --git a/test/command/suite/schema/tables/token_filters.test b/test/command/suite/schema/tables/token_filters.test
@@ -0,0 +1,7 @@
+plugin_register token_filters/stop_word
+
+table_create Terms TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --token_filters TokenFilterStopWord
+
+schema
diff --git a/test/command/suite/schema/tables/tokenizer.expected b/test/command/suite/schema/tables/tokenizer.expected
@@ -176,7 +176,10 @@ schema
         },
         "value_type": null,
         "tokenizer": "TokenBigram",
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       }
     }
   }

diff --git a/test/command/suite/schema/tables/type/array.expected b/test/command/suite/schema/tables/type/array.expected
@@ -173,7 +173,10 @@ schema
         "key_type": null,
         "value_type": null,
         "tokenizer": null,
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       }
     }
   }

diff --git a/test/command/suite/schema/tables/type/hash_table.expected b/test/command/suite/schema/tables/type/hash_table.expected
@@ -176,7 +176,10 @@ schema
         },
         "value_type": null,
         "tokenizer": null,
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       }
     }
   }

diff --git a/test/command/suite/schema/tables/value_type/reference.expected b/test/command/suite/schema/tables/value_type/reference.expected
@@ -178,7 +178,10 @@ schema
           "type": "reference"
         },
         "tokenizer": null,
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       },
       "Users": {
         "name": "Users",
@@ -189,7 +192,10 @@ schema
         },
         "value_type": null,
         "tokenizer": null,
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       }
     }
   }

diff --git a/test/command/suite/schema/tables/value_type/type.expected b/test/command/suite/schema/tables/value_type/type.expected
@@ -176,7 +176,10 @@ schema
           "type": "type"
         },
         "tokenizer": null,
-        "normalizer": null
+        "normalizer": null,
+        "token_filters": [
+
+        ]
       }
     }
   }