Navigation Menu

Skip to content

Commit

Permalink
index-check: improve
Browse files Browse the repository at this point in the history 
  * Support multiple columns
  * Fix a bug that nothing is checked in index content check
  * Add more tests
  • Loading branch information
@kou
kou committed Oct 30, 2017
1 parent 3216da3 commit 6421287
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 29 deletions.
2 changes: 1 addition & 1 deletion groonga-client.gemspec
View file
Expand Up @@ -49,7 +49,7 @@ Gem::Specification.new do |spec|

spec.add_runtime_dependency("gqtp", ">= 1.0.4")
spec.add_runtime_dependency("groonga-command", ">= 1.2.8")
spec.add_runtime_dependency("groonga-command-parser", ">= 1.0.7")
spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
spec.add_runtime_dependency("hashie")

spec.add_development_dependency("bundler")
Expand Down
70 changes: 44 additions & 26 deletions lib/groonga/client/command-line/groonga-client-index-check.rb
View file
Expand Up @@ -145,35 +145,39 @@ def list_tokens(table_name)
end
end

def verify_tokens(table_name, old_column, new_column, tokens)
broken_index_tokens = []
def verify_tokens(source_table, table_name, old_column, new_column, tokens)
full_old_column = "#{table_name}.#{old_column}"
full_new_column = "#{table_name}.#{new_column}"
tokens.each do |token|
query = Groonga::Client::ScriptSyntax.format_string(token)
case token
when String
value = Groonga::Client::ScriptSyntax.format_string(token)
else
value = token
end
old_response = execute_command(:select,
:table => table_name,
:match_columns => old_column,
:query => query,
:table => source_table,
:filter => "#{full_old_column} @ #{value}",
:output_columns => "_id",
:limit => "-1",
:sort_keys => "_id")
new_response = execute_command(:select,
:table => table_name,
:match_columns => new_column,
:query => query,
:table => source_table,
:filter => "#{full_new_column} @ #{value}",
:output_columns => "_id",
:limit => "-1",
:sort_keys => "_id")
old_response_ids = old_response.records.collect do |value|
value["_id"]
old_response_ids = old_response.records.collect do |record|
record["_id"]
end
new_response_ids = new_response.records.collect do |value|
value["_id"]
new_response_ids = new_response.records.collect do |record|
record["_id"]
end
if old_response_ids != new_response_ids
broken_index_tokens << token
return token
end
end
broken_index_tokens
nil
end

def check_content(index_column)
Expand All @@ -183,27 +187,41 @@ def check_content(index_column)
column_name = index_column["name"]
suffix = Time.now.strftime("%Y%m%d%H%M%S_%N")
new_column_name = "#{column_name}_#{suffix}"
type, source = index_column.sources.first.split(".")
source_table = nil
source_columns = []
index_column.sources.each do |source|
if source.include?(".")
source_table, source_column = source.split(".")
source_columns << source_column
else
source_table = source
source_columns << "_key"
end
end
flags = index_column["flags"].split("|")
flags.delete("PERSISTENT")
column_create(table_name,
new_column_name,
flags.join("|"),
type,
source)
source_table,
source_columns.join(","))
begin
tokens = list_tokens(table_name)
broken_index_tokens = verify_tokens(table_name, column_name,
new_column_name, tokens)
broken_token = verify_tokens(source_table,
table_name,
column_name,
new_column_name,
tokens)
if broken_token
$stderr.puts("Broken: #{table_name}.#{column_name}: " +
"<#{broken_token}>")
false
else
true
end
ensure
column_remove(table_name, new_column_name)
end
if broken_index_tokens.empty?
true
else
$stderr.puts("Broken: #{table_name}.#{column_name}")
false
end
end
end
end
Expand Down
141 changes: 139 additions & 2 deletions test/command-line/test-index-check.rb
View file
Expand Up @@ -48,7 +48,7 @@ def test_source
OUTPUT

assert_equal([false, "", error_output],
index_check("--method=source", "Terms.memos_content"))
index_check("--method=source"))
end

sub_test_case("content") do
Expand All @@ -72,7 +72,144 @@ def test_valid
COMMANDS

assert_equal([true, "", ""],
index_check("--method=content", "Terms.memos_content"))
index_check("--method=content"))
end

def test_all
restore(<<-COMMANDS)
table_create Memos TABLE_HASH_KEY ShortText
column_create Memos content COLUMN_SCALAR Text
table_create Terms TABLE_PAT_KEY ShortText \
--normalizer NormalizerAuto \
--default_tokenizer TokenBigram
column_create Terms memos_content1 \
COLUMN_INDEX|WITH_POSITION \
Memos content
column_create Terms memos_content2 \
COLUMN_INDEX|WITH_POSITION \
Memos content
load --table Memos
[
["_key", "content"],
["groonga", "Groonga is fast"]
]
delete --table Terms --key is
COMMANDS

assert_equal([
false,
"",
"Broken: Terms.memos_content1: <is>\n" +
"Broken: Terms.memos_content2: <is>\n",
],
index_check("--method=content"))
end


def test_specify
restore(<<-COMMANDS)
table_create Memos TABLE_HASH_KEY ShortText
column_create Memos content COLUMN_SCALAR Text
table_create Terms TABLE_PAT_KEY ShortText \
--normalizer NormalizerAuto \
--default_tokenizer TokenBigram
column_create Terms memos_content1 \
COLUMN_INDEX|WITH_POSITION \
Memos content
column_create Terms memos_content2 \
COLUMN_INDEX|WITH_POSITION \
Memos content
load --table Memos
[
["_key", "content"],
["groonga", "Groonga is fast"]
]
delete --table Terms --key is
COMMANDS

assert_equal([
false,
"",
"Broken: Terms.memos_content1: <is>\n",
],
index_check("--method=content", "Terms.memos_content1"))
end

def test_broken_single_column
restore(<<-COMMANDS)
table_create Memos TABLE_HASH_KEY ShortText
column_create Memos content COLUMN_SCALAR Text
table_create Terms TABLE_PAT_KEY ShortText \
--normalizer NormalizerAuto \
--default_tokenizer TokenBigram
column_create Terms memos_content \
COLUMN_INDEX|WITH_POSITION \
Memos content
load --table Memos
[
{"_key": "groonga", "content": "Groonga is fast"}
]
delete Terms --key is
COMMANDS

assert_equal([false, "", "Broken: Terms.memos_content: <is>\n"],
index_check("--method=content"))
end

def test_key
restore(<<-COMMANDS)
table_create Memos TABLE_HASH_KEY ShortText
table_create Terms TABLE_PAT_KEY ShortText \
--normalizer NormalizerAuto \
--default_tokenizer TokenBigram
column_create Terms memos_key \
COLUMN_INDEX|WITH_POSITION \
Memos _key
load --table Memos
[
{"_key": "groonga"}
]
delete Terms --key groonga
COMMANDS

assert_equal([false, "", "Broken: Terms.memos_key: <groonga>\n"],
index_check("--method=content"))
end

def test_broken_multiple_column
restore(<<-COMMANDS)
table_create Memos TABLE_HASH_KEY ShortText
column_create Memos content COLUMN_SCALAR Text
table_create Terms TABLE_PAT_KEY ShortText \
--normalizer NormalizerAuto \
--default_tokenizer TokenBigram
column_create Terms memos \
COLUMN_INDEX|WITH_SECTION|WITH_POSITION \
Memos _key,content
load --table Memos
[
{"_key": "groonga", "content": "Groonga is fast"}
]
delete Terms --key is
COMMANDS

assert_equal([false, "", "Broken: Terms.memos: <is>\n"],
index_check("--method=content"))
end
end
end

0 comments on commit 6421287

Please sign in to comment.