Skip to content

Commit

Permalink
select logical_select: add more query logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@kou
kou committed May 13, 2019
1 parent ec54710 commit 5bcacb6
Show file tree
Hide file tree
Showing 49 changed files with 366 additions and 110 deletions.
94 changes: 63 additions & 31 deletions lib/proc/proc_select.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1366,7 +1366,9 @@ static void
grn_select_apply_columns(grn_ctx *ctx,
grn_select_data *data,
grn_obj *table,
grn_hash *columns)
grn_hash *columns,
const char *log_tag_prefix,
const char *query_log_tag_prefix)
{
grn_hash_cursor *columns_cursor;

Expand Down Expand Up @@ -1394,7 +1396,8 @@ grn_select_apply_columns(grn_ctx *ctx,
if (!column) {
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] failed to create column: %s",
"%s[column][%s][%.*s] failed to create column: %s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand All @@ -1407,8 +1410,9 @@ grn_select_apply_columns(grn_ctx *ctx,
grn_obj_close(ctx, column);
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] "
"%s[column][%s][%.*s] "
"failed to create expression to compute value: %s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand All @@ -1428,8 +1432,9 @@ grn_select_apply_columns(grn_ctx *ctx,
grn_obj_close(ctx, column);
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] "
"%s[column][%s][%.*s] "
"failed to parse value: <%.*s>: %s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand Down Expand Up @@ -1460,8 +1465,9 @@ grn_select_apply_columns(grn_ctx *ctx,
grn_obj_close(ctx, column);
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] "
"%s[column][%s][%.*s] "
"failed to parse sort keys: %s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand Down Expand Up @@ -1491,8 +1497,9 @@ grn_select_apply_columns(grn_ctx *ctx,
}
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] "
"%s[column][%s][%.*s] "
"failed to parse group keys: %s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand Down Expand Up @@ -1532,9 +1539,10 @@ grn_select_apply_columns(grn_ctx *ctx,
grn_obj_close(ctx, column);
GRN_PLUGIN_ERROR(ctx,
GRN_INVALID_ARGUMENT,
"[select][column][%s][%.*s] "
"%s[column][%s][%.*s] "
"failed to apply expression to generate column values: "
"%s",
log_tag_prefix,
grn_column_stage_name(column_data->stage),
(int)(column_data->label.length),
column_data->label.value,
Expand All @@ -1546,7 +1554,8 @@ grn_select_apply_columns(grn_ctx *ctx,
grn_obj_close(ctx, expression);

GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "columns[%.*s](%d)",
":", "%scolumns[%.*s](%d)",
query_log_tag_prefix ? query_log_tag_prefix : "",
(int)(column_data->label.length),
column_data->label.value,
grn_table_size(ctx, table));
Expand All @@ -1572,7 +1581,9 @@ grn_select_apply_initial_columns(grn_ctx *ctx,
grn_select_apply_columns(ctx,
data,
data->tables.initial,
data->columns.initial);
data->columns.initial,
"[select]",
NULL);

return ctx->rc == GRN_SUCCESS;
}
Expand Down Expand Up @@ -1624,7 +1635,9 @@ grn_select_apply_filtered_columns(grn_ctx *ctx,
grn_select_apply_columns(ctx,
data,
data->tables.result,
data->columns.filtered);
data->columns.filtered,
"[select]",
NULL);

return ctx->rc == GRN_SUCCESS;
}
Expand Down Expand Up @@ -1784,9 +1797,8 @@ grn_select_apply_adjuster(grn_ctx *ctx,
grn_obj_unlink(ctx, adjuster);

GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "%s%sadjust(%d)",
":", "%sadjust(%d)",
query_log_tag_prefix ? query_log_tag_prefix : "",
query_log_tag_prefix ? "." : "",
grn_table_size(ctx, result));

return GRN_TRUE;
Expand Down Expand Up @@ -1915,7 +1927,11 @@ grn_select_sort(grn_ctx *ctx,
grn_table_sort_key_close(ctx, keys, n_keys);

GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "sort(%d)", data->limit);
":",
"sort(%d): %.*s",
data->limit,
(int)(data->sort_keys.length),
data->sort_keys.value);

return ctx->rc == GRN_SUCCESS;
}
Expand Down Expand Up @@ -2042,7 +2058,9 @@ grn_select_apply_output_columns(grn_ctx *ctx,
grn_select_apply_columns(ctx,
data,
data->tables.sorted,
data->columns.output);
data->columns.output,
"[select]",
NULL);

return ctx->rc == GRN_SUCCESS;
}
Expand Down Expand Up @@ -2314,11 +2332,11 @@ grn_select_drilldown_execute(grn_ctx *ctx,
GRN_TEXT_PUTC(ctx, &log_tag_prefix, '\0');
if (data->drilldown.keys.length == 0) {
grn_text_printf(ctx, &query_log_tag_prefix,
"drilldowns[%.*s]",
"drilldowns[%.*s].",
(int)(drilldown->label.length),
drilldown->label.value);
} else {
GRN_TEXT_SETS(ctx, &query_log_tag_prefix, "drilldown");
GRN_TEXT_SETS(ctx, &query_log_tag_prefix, "drilldown.");
}
GRN_TEXT_PUTC(ctx, &query_log_tag_prefix, '\0');

Expand Down Expand Up @@ -2428,11 +2446,19 @@ grn_select_drilldown_execute(grn_ctx *ctx,
goto exit;
}

GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "%.*s(%u)",
(int)(GRN_TEXT_LEN(&query_log_tag_prefix) - 2),
GRN_TEXT_VALUE(&query_log_tag_prefix),
grn_table_size(ctx, result->table));

if (drilldown->columns.initial) {
grn_select_apply_columns(ctx,
data,
result->table,
drilldown->columns.initial);
drilldown->columns.initial,
GRN_TEXT_VALUE(&log_tag_prefix),
GRN_TEXT_VALUE(&query_log_tag_prefix));
}

if (drilldown->filter.length > 0) {
Expand Down Expand Up @@ -2493,6 +2519,11 @@ grn_select_drilldown_execute(grn_ctx *ctx,
goto exit;
}
grn_obj_close(ctx, expression);

GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "%sfilter(%u)",
GRN_TEXT_VALUE(&query_log_tag_prefix),
grn_table_size(ctx, drilldown->filtered_result));
}

{
Expand All @@ -2511,20 +2542,6 @@ grn_select_drilldown_execute(grn_ctx *ctx,
GRN_TEXT_VALUE(&query_log_tag_prefix));
}

{
unsigned int n_hits;

if (drilldown->filtered_result) {
n_hits = grn_table_size(ctx, drilldown->filtered_result);
} else {
n_hits = grn_table_size(ctx, result->table);
}
GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "%s(%u)",
GRN_TEXT_VALUE(&query_log_tag_prefix),
n_hits);
}

exit :
GRN_OBJ_FIN(ctx, &log_tag_prefix);
GRN_OBJ_FIN(ctx, &query_log_tag_prefix);
Expand Down Expand Up @@ -2877,6 +2894,21 @@ grn_select_output_drilldowns(grn_ctx *ctx,
if (sorted) {
grn_table_sort(ctx, target_table, offset, limit,
sorted, sort_keys, n_sort_keys);
if (is_labeled) {
GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "drilldowns[%.*s].sort(%d): %.*s",
(int)(drilldown->label.length),
drilldown->label.value,
limit,
(int)(drilldown->sort_keys.length),
drilldown->sort_keys.value);
} else {
GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
":", "drilldown.sort(%d): %.*s",
limit,
(int)(drilldown->sort_keys.length),
drilldown->sort_keys.value);
}
data->output.formatter->drilldown_label(ctx, data, drilldown);
if (!grn_select_output_columns(ctx,
data,
Expand Down
63 changes: 45 additions & 18 deletions plugins/sharding/dynamic_columns.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ class DynamicColumns
include Enumerable

class << self
def parse(input)
def parse(tag, input)
initial_contexts = {}
filtered_contexts = {}
output_contexts = {}
Expand All @@ -21,7 +21,9 @@ def parse(input)
else
next
end
contexts[label] = DynamicColumnExecuteContext.new(label, arguments)
contexts[label] = DynamicColumnExecuteContext.new(tag,
label,
arguments)
end

new(DynamicColumnExecuteContexts.new(initial_contexts),
Expand Down Expand Up @@ -125,15 +127,26 @@ def apply(contexts, targets, options)
end

if options.fetch(:normal, true)
targets.each do |result_set, options|
targets.each do |result_set, target_options|
normal_contexts.each do |context|
context.apply(result_set, options)
global_options = nil
if options and options[:query_log_prefix]
global_options = {
query_log_prefix: options[:query_log_prefix],
}
end
if target_options and global_options
target_options = global_options.merge(target_options)
elsif global_options
target_options = global_options
end
context.apply(result_set, target_options)
end
end
end
if options.fetch(:window_function, true)
window_function_contexts.each do |context|
context.apply_window_function(targets)
context.apply_window_function(targets, options)
end
end
end
Expand Down Expand Up @@ -183,6 +196,7 @@ def tsort_each_child(context, &block)
end

class DynamicColumnExecuteContext
include QueryLoggable
include KeysParsable

attr_reader :label
Expand All @@ -192,7 +206,8 @@ class DynamicColumnExecuteContext
attr_reader :value
attr_reader :window_sort_keys
attr_reader :window_group_keys
def initialize(label, arguments)
def initialize(tag, label, arguments)
@tag = tag
@label = label
@stage = arguments["stage"]
@type = parse_type(arguments["type"])
Expand All @@ -210,22 +225,29 @@ def window_function?
def apply(table, options=nil)
return if table.find_column(@label)
column = table.create_column(@label, @flags, @type)
return if table.empty?

condition = nil
condition = options[:condition] if options
expression = Expression.create(table)
begin
expression.parse(@value)
expression.condition = condition if condition
table.apply_expression(column, expression)
ensure
expression.close
p options
query_log_prefix = nil
query_log_prefix = options[:query_log_prefix] if options
unless table.empty?
condition = nil
condition = options[:condition] if options
expression = Expression.create(table)
begin
expression.parse(@value)
expression.condition = condition if condition
table.apply_expression(column, expression)
ensure
expression.close
end
end
query_logger.log(:size, ":",
"#{query_log_prefix}columns[#{@label}](#{table.size})")
end

def apply_window_function(targets)
def apply_window_function(targets, options=nil)
executor = WindowFunctionExecutor.new
n_records = 0
begin
executor.source = @value
executor.sort_keys = @window_sort_keys.join(", ")
Expand All @@ -240,12 +262,17 @@ def apply_window_function(targets)
executor.add_context_table(table)
else
executor.add_table(table)
n_records += table.size
end
end
executor.execute
ensure
executor.close
end
query_log_prefix = nil
query_log_prefix = options[:query_log_prefix] if options
query_logger.log(:size, ":",
"#{query_log_prefix}columns[#{@label}](#{n_records})")
end

private
Expand Down Expand Up @@ -276,7 +303,7 @@ def context_target?(options)
end

def error_message_tag
"[logical_select][columns][#{@stage}][#{@label}]"
"#{@tag}[columns][#{@stage}][#{@label}]"
end
end
end
Expand Down
4 changes: 2 additions & 2 deletions plugins/sharding/logical_count.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def cache_key(input)
key << "#{input[:max_border]}\0"
key << "#{input[:filter]}\0"
key << "#{input[:post_filter]}\0"
dynamic_columns = DynamicColumns.parse(input)
dynamic_columns = DynamicColumns.parse("[logical_count]", input)
key << dynamic_columns.cache_key
key
end
Expand All @@ -74,7 +74,7 @@ class Counter
def initialize(input, target_range)
@filter = input[:filter]
@post_filter = input[:post_filter]
@dynamic_columns = DynamicColumns.parse(input)
@dynamic_columns = DynamicColumns.parse("[logical_count]", input)
@target_range = target_range
@contexts = []
@temporary_tables = []
Expand Down
Loading

0 comments on commit 5bcacb6

Please sign in to comment.