Navigation Menu

Skip to content

Commit

Permalink
logical_range_filter: support negative limit and offset
Browse files Browse the repository at this point in the history
  • Loading branch information
kou committed Apr 2, 2015
1 parent 408fdd7 commit d880dc6
Show file tree
Hide file tree
Showing 9 changed files with 448 additions and 7 deletions.
44 changes: 37 additions & 7 deletions plugins/sharding/logical_range_filter.rb
Expand Up @@ -35,8 +35,19 @@ def run_body(input)
if first_result_set
writer.write_table_columns(first_result_set, output_columns)
end
limit = context.limit
if limit < 0
n_records = result_sets.inject(0) do |n, result_set|
n + result_set.size
end
limit = n_records + limit + 1
end
options = {}
result_sets.each do |result_set|
writer.write_table_records(result_set, output_columns)
options[:limit] = limit
writer.write_table_records(result_set, output_columns, options)
limit -= result_set.size
break if limit <= 0
end
end
ensure
Expand All @@ -63,7 +74,7 @@ def initialize(input)
@offset = (@input[:offset] || 0).to_i
@limit = (@input[:limit] || 10).to_i

@current_offset = 0
@current_offset = @offset
@current_limit = @limit

@result_sets = []
Expand Down Expand Up @@ -207,7 +218,12 @@ def execute

private
def use_range_index?(range_index)
required_n_records = @context.current_offset + @context.current_limit
current_limit = @context.current_limit
if current_limit < 0
return false
end

required_n_records = @context.current_offset + current_limit
max_n_records = @table.size
if max_n_records <= required_n_records
return false
Expand Down Expand Up @@ -305,8 +321,13 @@ def filter_by_range(range_index,
:flags => flags) do |table_cursor|
options = {
:offset => @context.current_offset,
:limit => @context.current_limit,
}
current_limit = @context.current_limit
if current_limit < 0
options[:limit] = data_table.size
else
options[:limit] = current_limit
end
if @filter
create_expression(data_table) do |expression|
expression.parse(@filter)
Expand Down Expand Up @@ -335,7 +356,9 @@ def filter_by_range(range_index,
if @context.current_offset > 0
@context.current_offset = 0
end
@context.current_limit -= result_set.size
if @context.current_limit > 0
@context.current_limit -= result_set.size
end
@result_sets << result_set
end

Expand Down Expand Up @@ -389,14 +412,21 @@ def sort_result_set(result_set)
:order => @context.order,
},
]
if @context.current_limit > 0
limit = @context.current_limit
else
limit = result_set.size
end
sorted_result_set = result_set.sort(sort_keys,
:offset => @context.current_offset,
:limit => @context.current_limit)
:limit => limit)
@result_sets << sorted_result_set
if @context.current_offset > 0
@context.current_offset = 0
end
@context.current_limit -= sorted_result_set.size
if @context.current_limit > 0
@context.current_limit -= sorted_result_set.size
end
end
end
end
Expand Down
@@ -0,0 +1,75 @@
load --table Logs_20150203
[
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Start"},
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Shutdown"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Start"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
load --table Logs_20150204
[
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Start"},
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Start"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Shutdown"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Start"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],6]
load --table Logs_20150205
[
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Start"},
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Start"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
logical_range_filter Logs timestamp --filter 'message == "Shutdown"' --min "2015-02-03 23:59:59" --min_border "include" --max "2015-02-05 00:00:00" --max_border "include" --limit -1
[
[
0,
0.0,
0.0
],
[
[
[
"memo",
"ShortText"
],
[
"message",
"Text"
],
[
"timestamp",
"Time"
]
],
[
"2015-02-03 23:59:59",
"Shutdown",
1422975599.0
],
[
"2015-02-04 00:00:00",
"Shutdown",
1422975600.0
],
[
"2015-02-04 00:00:01",
"Shutdown",
1422975601.0
],
[
"2015-02-04 23:59:59",
"Shutdown",
1423061999.0
],
[
"2015-02-05 00:00:00",
"Shutdown",
1423062000.0
]
]
]
@@ -0,0 +1,36 @@
#@include fixture/sharding/logical_range_filter/index/schema.grn

load --table Logs_20150203
[
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Start"},
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Shutdown"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Start"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Shutdown"}
]

load --table Logs_20150204
[
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Start"},
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Start"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Shutdown"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Start"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Shutdown"}
]

load --table Logs_20150205
[
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Start"},
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Start"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Shutdown"}
]

logical_range_filter Logs timestamp \
--filter 'message == "Shutdown"' \
--min "2015-02-03 23:59:59" \
--min_border "include" \
--max "2015-02-05 00:00:00" \
--max_border "include" \
--limit -1

@@ -0,0 +1,65 @@
load --table Logs_20150203
[
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Start"},
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Shutdown"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Start"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
load --table Logs_20150204
[
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Start"},
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Start"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Shutdown"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Start"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],6]
load --table Logs_20150205
[
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Start"},
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Start"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
logical_range_filter Logs timestamp --filter 'message == "Shutdown"' --min "2015-02-03 23:59:59" --min_border "include" --max "2015-02-05 00:00:00" --max_border "include" --offset 2 --limit -1
[
[
0,
0.0,
0.0
],
[
[
[
"memo",
"ShortText"
],
[
"message",
"Text"
],
[
"timestamp",
"Time"
]
],
[
"2015-02-04 00:00:01",
"Shutdown",
1422975601.0
],
[
"2015-02-04 23:59:59",
"Shutdown",
1423061999.0
],
[
"2015-02-05 00:00:00",
"Shutdown",
1423062000.0
]
]
]
@@ -0,0 +1,37 @@
#@include fixture/sharding/logical_range_filter/index/schema.grn

load --table Logs_20150203
[
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Start"},
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Shutdown"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Start"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Shutdown"}
]

load --table Logs_20150204
[
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Start"},
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Start"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Shutdown"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Start"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Shutdown"}
]

load --table Logs_20150205
[
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Start"},
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Start"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Shutdown"}
]

logical_range_filter Logs timestamp \
--filter 'message == "Shutdown"' \
--min "2015-02-03 23:59:59" \
--min_border "include" \
--max "2015-02-05 00:00:00" \
--max_border "include" \
--offset 2 \
--limit -1

@@ -0,0 +1,65 @@
load --table Logs_20150203
[
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Start"},
{"timestamp": "2015-02-03 23:59:58", "memo": "2015-02-03 23:59:58", "message": "Shutdown"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Start"},
{"timestamp": "2015-02-03 23:59:59", "memo": "2015-02-03 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
load --table Logs_20150204
[
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Start"},
{"timestamp": "2015-02-04 00:00:00", "memo": "2015-02-04 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Start"},
{"timestamp": "2015-02-04 00:00:01", "memo": "2015-02-04 00:00:01", "message": "Shutdown"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Start"},
{"timestamp": "2015-02-04 23:59:59", "memo": "2015-02-04 23:59:59", "message": "Shutdown"}
]
[[0,0.0,0.0],6]
load --table Logs_20150205
[
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Start"},
{"timestamp": "2015-02-05 00:00:00", "memo": "2015-02-05 00:00:00", "message": "Shutdown"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Start"},
{"timestamp": "2015-02-05 00:00:01", "memo": "2015-02-05 00:00:01", "message": "Shutdown"}
]
[[0,0.0,0.0],4]
logical_range_filter Logs timestamp --filter 'message == "Shutdown"' --min "2015-02-03 23:59:59" --min_border "include" --max "2015-02-05 00:00:00" --max_border "include" --limit -3
[
[
0,
0.0,
0.0
],
[
[
[
"memo",
"ShortText"
],
[
"message",
"Text"
],
[
"timestamp",
"Time"
]
],
[
"2015-02-03 23:59:59",
"Shutdown",
1422975599.0
],
[
"2015-02-04 00:00:00",
"Shutdown",
1422975600.0
],
[
"2015-02-04 00:00:01",
"Shutdown",
1422975601.0
]
]
]

0 comments on commit d880dc6

Please sign in to comment.