Navigation Menu

Skip to content

Commit

Permalink
tokenize: support mode and token_filters
Browse files Browse the repository at this point in the history
  • Loading branch information
kou committed Jan 26, 2015
1 parent 6aa3edd commit 9c03966
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 2 deletions.
21 changes: 20 additions & 1 deletion lib/groonga/command/tokenize.rb
Expand Up @@ -33,6 +33,8 @@ def parameter_names
:string,
:normalizer,
:flags,
:mode,
:token_filters,
]
end
end
Expand All @@ -55,7 +57,7 @@ def normalizer
self[:normalizer]
end

# @return [Array<String>] An array of flags specified in `flags`
# @return [Array<String>] An array of flag specified in `flags`
# parameter value. This array is extracted by parsing `flags`
# parameter value. If `flags` parameter value is nil or empty,
# an empty array is returned.
Expand All @@ -64,6 +66,23 @@ def normalizer
def flags
@flags ||= (self[:flags] || "").split(/\s*[| ]\s*/)
end

# @return [String] `mode` parameter value.
#
# @since 1.1.0
def mode
self[:mode]
end

# @return [Array<String>] An array of token filter specified in
# `token_filters` parameter value. This array is extracted by
# parsing `token_filters` parameter value. If `token_filters`
# parameter value is nil or empty, an empty array is returned.
#
# @since 1.1.0
def token_filters
@token_filters ||= (self[:token_filters] || "").split(/\s*,\s*/)
end
end
end
end
41 changes: 40 additions & 1 deletion test/command/test-tokenize.rb
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
# Copyright (C) 2013-2015 Kouhei Sutou <kou@clear-code.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
Expand Down Expand Up @@ -30,19 +30,25 @@ def test_ordered_arguments
string = "groonga ruby linux"
normalizer = "NormalizerAuto"
flags = "NONE"
mode = "ADD"
token_filters = "TokenFilterStem"

ordered_arguments = [
tokenizer,
string,
normalizer,
flags,
mode,
token_filters
]
command = tokenize_command({}, ordered_arguments)
assert_equal({
:tokenizer => tokenizer,
:string => string,
:normalizer => normalizer,
:flags => flags,
:mode => mode,
:token_filters => token_filters,
},
command.arguments)
end
Expand Down Expand Up @@ -95,4 +101,37 @@ def test_spaces_around_separator
assert_equal(["NONE", "ENABLE_TOKENIZED_DELIMITER"], command.flags)
end
end

class ModeTest < self
def test_reader
command = tokenize_command(:mode => "ADD")
assert_equal("ADD", command.mode)
end
end

class TokenFiltersTest < self
def test_nil
command = tokenize_command
assert_equal([], command.token_filters)
end

def test_empty
command = tokenize_command(:token_filters => "")
assert_equal([], command.token_filters)
end

def test_comma_separator
token_filters = "TokenFilterStem,TokenFilterStopWord"
command = tokenize_command(:token_filters => token_filters)
assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
command.token_filters)
end

def test_spaces_around_separator
token_filters = "TokenFilterStem , TokenFilterStopWord"
command = tokenize_command(:token_filters => token_filters)
assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
command.token_filters)
end
end
end

0 comments on commit 9c03966

Please sign in to comment.