diff --git a/CHANGELOG.md b/CHANGELOG.md index bd698e30..f8cb332c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed * `checks` is disabled for internal functions by default +* `limit` option is renamed to `first` +* Reverse pagination (negative `first`) is supported ## [0.1.0] - 2020-09-23 diff --git a/README.md b/README.md index ae27a93c..357c69bc 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,9 @@ where: * `space_name` (`string`) - name of the space * `conditions` (`?table`) - array of [select conditions](#select-conditions) * `opts`: - * `limit` (`?number`) - the maximum limit of the objects to return + * `first` (`?number`) - the maximum count of the objects to return. + If negative value is specified, the last objects are returned + (`after` option is required in this case). * `after` (`?table`) - object after which objects should be selected * `batch_size` (`?number`) - number of tuples to process per one request to storage * `timeout` (`?number`) - `vshard.call` timeout (in seconds) @@ -261,7 +263,8 @@ crud.select('customers', {{'<=', 'age', 35}}) ### Pairs You can iterate across a distributed space using the `crud.pairs` function. -Its arguments are the same as [`crud.select`](#select) arguments. +Its arguments are the same as [`crud.select`](#select) arguments, +but negative `first` values aren't allowed. **Example:** diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 96e71739..94da965f 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -173,4 +173,26 @@ function utils.unflatten_rows(rows, metadata) return result end +local inverted_tarantool_iters = { + [box.index.EQ] = box.index.REQ, + [box.index.GT] = box.index.LT, + [box.index.GE] = box.index.LE, + [box.index.LT] = box.index.GT, + [box.index.LE] = box.index.GE, + [box.index.REQ] = box.index.EQ, +} + +function utils.invert_tarantool_iter(iter) + local inverted_iter = inverted_tarantool_iters[iter] + assert(inverted_iter ~= nil, "Unsupported Tarantool iterator: " .. tostring(iter)) + return inverted_iter +end + +function utils.reverse_inplace(t) + for i = 1,#t - 1 do + t[i], t[#t - i + 1] = t[#t - i + 1], t[i] + end + return t +end + return utils diff --git a/crud/select.lua b/crud/select.lua index 247e0eb2..95891873 100644 --- a/crud/select.lua +++ b/crud/select.lua @@ -119,7 +119,7 @@ end local function build_select_iterator(space_name, user_conditions, opts) dev_checks('string', '?table', { after = '?table', - limit = '?number', + first = '?number', timeout = '?number', batch_size = '?number', }) @@ -132,10 +132,6 @@ local function build_select_iterator(space_name, user_conditions, opts) local batch_size = opts.batch_size or DEFAULT_BATCH_SIZE - if opts.limit ~= nil and opts.limit < 0 then - return nil, SelectError:new("limit should be >= 0") - end - -- check conditions local conditions, err = select_conditions.parse(user_conditions) if err ~= nil then @@ -153,25 +149,26 @@ local function build_select_iterator(space_name, user_conditions, opts) end local space_format = space:format() + -- set after tuple + local after_tuple = utils.flatten(opts.after, space_format) + -- plan select local plan, err = select_plan.new(space, conditions, { - limit = opts.limit, + first = opts.first, + after_tuple = after_tuple, }) if err ~= nil then return nil, SelectError:new("Failed to plan select: %s", err) end - -- set limit and replicasets to select from + -- set replicasets to select from local replicasets_to_select = replicasets if plan.sharding_key ~= nil then replicasets_to_select = get_replicasets_by_sharding_key(plan.sharding_key) end - -- set after tuple - local after_tuple = utils.flatten(opts.after, space_format) - -- generate tuples comparator local scan_index = space.index[plan.index_id] local primary_index = space.index[0] @@ -191,7 +188,6 @@ local function build_select_iterator(space_name, user_conditions, opts) comparator = tuples_comparator, plan = plan, - after_tuple = after_tuple, batch_size = batch_size, replicasets = replicasets_to_select, @@ -205,16 +201,20 @@ end function select_module.pairs(space_name, user_conditions, opts) checks('string', '?table', { after = '?table', - limit = '?number', + first = '?number', timeout = '?number', batch_size = '?number', }) opts = opts or {} + if opts.first ~= nil and opts.first < 0 then + error(string.format("Negative first isn't allowed for pairs")) + end + local iter, err = build_select_iterator(space_name, user_conditions, { after = opts.after, - limit = opts.limit, + first = opts.first, timeout = opts.timeout, batch_size = opts.batch_size, }) @@ -247,16 +247,22 @@ end function select_module.call(space_name, user_conditions, opts) checks('string', '?table', { after = '?table', - limit = '?number', + first = '?number', timeout = '?number', batch_size = '?number', }) opts = opts or {} + if opts.first ~= nil and opts.first < 0 then + if opts.after == nil then + return nil, SelectError:new("Negative first should be specified only with after option") + end + end + local iter, err = build_select_iterator(space_name, user_conditions, { after = opts.after, - limit = opts.limit, + first = opts.first, timeout = opts.timeout, batch_size = opts.batch_size, }) @@ -268,16 +274,20 @@ function select_module.call(space_name, user_conditions, opts) local tuples = {} while iter:has_next() do - local obj, err = iter:get() + local tuple, err = iter:get() if err ~= nil then return nil, SelectError:new("Failed to get next object: %s", err) end - if obj == nil then + if tuple == nil then break end - table.insert(tuples, obj) + table.insert(tuples, tuple) + end + + if opts.first ~= nil and opts.first < 0 then + utils.reverse_inplace(tuples) end return { diff --git a/crud/select/comparators.lua b/crud/select/comparators.lua index c06408ce..9be42b4d 100644 --- a/crud/select/comparators.lua +++ b/crud/select/comparators.lua @@ -177,7 +177,7 @@ local array_cmp_funcs_by_operators = { --]=] function comparators.get_cmp_operator(tarantool_iter) local cmp_operator = cmp_operators_by_tarantool_iter[tarantool_iter] - assert(cmp_operator ~= nil, 'Unsupported Tarantool iterator %q', tarantool_iter) + assert(cmp_operator ~= nil, 'Unsupported Tarantool iterator: ' .. tostring(tarantool_iter)) return cmp_operator end diff --git a/crud/select/iterator.lua b/crud/select/iterator.lua index 62c0552d..48eed1b3 100644 --- a/crud/select/iterator.lua +++ b/crud/select/iterator.lua @@ -20,7 +20,6 @@ function Iterator.new(opts) iteration_func = 'function', plan = 'table', - after_tuple = '?table', batch_size = 'number', replicasets = 'table', @@ -34,7 +33,7 @@ function Iterator.new(opts) iteration_func = opts.iteration_func, plan = opts.plan, - after_tuple = opts.after_tuple, + timeout = opts.timeout, replicasets = table.copy(opts.replicasets), @@ -56,7 +55,7 @@ function Iterator.new(opts) setmetatable(iter, Iterator) - iter:_update_replicasets_tuples(iter.after_tuple) + iter:_update_replicasets_tuples(iter.plan.after_tuple) return iter end diff --git a/crud/select/plan.lua b/crud/select/plan.lua index 6319b623..c9de7a8d 100644 --- a/crud/select/plan.lua +++ b/crud/select/plan.lua @@ -1,6 +1,7 @@ local errors = require('errors') local select_conditions = require('crud.select.conditions') +local utils = require('crud.common.utils') local dev_checks = require('crud.common.dev_checks') local select_plan = {} @@ -86,8 +87,10 @@ end function select_plan.new(space, conditions, opts) dev_checks('table', '?table', { - limit = '?number', + first = '?number', + after_tuple = '?table', }) + conditions = conditions ~= nil and conditions or {} opts = opts or {} @@ -135,8 +138,26 @@ function select_plan.new(space, conditions, opts) scan_value = {} end - -- set total_tuples_count - local total_tuples_count = opts.limit + -- handle opts.first + local total_tuples_count + local scan_after_tuple = opts.after_tuple + + if opts.first ~= nil then + total_tuples_count = math.abs(opts.first) + + if opts.first < 0 then + scan_iter = utils.invert_tarantool_iter(scan_iter) + + -- scan condition becomes border consition + scan_condition_num = nil + + if scan_after_tuple ~= nil then + scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) + else + scan_value = nil + end + end + end local sharding_index = primary_index -- XXX: only sharding by primary key is supported @@ -156,6 +177,7 @@ function select_plan.new(space, conditions, opts) space_name = space_name, index_id = scan_index.id, scan_value = scan_value, + after_tuple = scan_after_tuple, scan_condition_num = scan_condition_num, iter = scan_iter, total_tuples_count = total_tuples_count, diff --git a/test/integration/pairs_test.lua b/test/integration/pairs_test.lua index 4dbd4ca1..9f22ab17 100644 --- a/test/integration/pairs_test.lua +++ b/test/integration/pairs_test.lua @@ -296,3 +296,28 @@ add('test_le_condition_with_index', function(g) t.assert_equals(err, nil) t.assert_equals(objects, get_by_ids(customers, {1})) -- in age order end) + +add('test_negative_first', function(g) + local customers = insert_customers(g,{ + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- negative first + t.assert_error_msg_contains("Negative first isn't allowed for pairs", function() + g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + crud.pairs('customers', nil, {first = -10}) + ]]) + end) +end) diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index 08aa5db5..a7d493d3 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -204,7 +204,7 @@ add('test_select_all', function(g) t.assert_equals(#objects, 0) end) -add('test_select_all_with_limit', function(g) +add('test_select_all_with_first', function(g) local customers = insert_customers(g, { { id = 1, name = "Elizabeth", last_name = "Jackson", @@ -223,35 +223,211 @@ add('test_select_all_with_limit', function(g) table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) - -- limit 2 + -- first 2 + local first = 2 local result, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') + local first = ... + local result, err = crud.select('customers', nil, { - limit = 2, + first = first, }) return result, err - ]]) + ]], {first}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, get_by_ids(customers, {1, 2})) - -- limit 0 + -- first 0 + local first = 0 local result, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') + local first = ... + local result, err = crud.select('customers', nil, { - limit = 0, + first = first, }) return result, err - ]]) + ]], {first}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(#objects, 0) end) +add('test_negative_first', function(g) + local customers = insert_customers(g, { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 11, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 22, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 44, city = "Chicago", + }, { + id = 5, name = "Jack", last_name = "Sparrow", + age = 55, city = "London", + }, { + id = 6, name = "William", last_name = "Terner", + age = 66, city = "Oxford", + }, { + id = 7, name = "Elizabeth", last_name = "Swan", + age = 77, city = "Cambridge", + }, { + id = 8, name = "Hector", last_name = "Barbossa", + age = 88, city = "London", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- negative first w/o after + local first = -10 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local first = ... + + local objects, err = crud.select('customers', nil, { + first = first, + }) + return objects, err + ]], {first}) + + t.assert_equals(result, nil) + t.assert_str_contains(err.err, "Negative first should be specified only with after option") + + -- no conditions + -- first -3 after 5 (batch_size is 1) + local first = -3 + local after = customers[5] + local batch_size = 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local first, after, batch_size = ... + + local objects, err = crud.select('customers', nil, { + first = first, + after = after, + batch_size = batch_size, + }) + return objects, err + ]], {first, after, batch_size}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, get_by_ids(customers, {2, 3, 4})) + + -- id >= 2 + -- first -2 after 5 (batch_size is 1) + local conditions = { + {'>=', 'id', 2}, + } + local first = -2 + local after = customers[5] + local batch_size = 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, first, after, batch_size = ... + + local objects, err = crud.select('customers', conditions, { + first = first, + after = after, + batch_size = batch_size, + }) + return objects, err + ]], {conditions, first, after, batch_size}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, get_by_ids(customers, {3, 4})) + + -- age >= 22 + -- first -2 after 5 (batch_size is 1) + local conditions = { + {'>=', 'age', 22}, + } + local first = -2 + local after = customers[5] + local batch_size = 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, first, after, batch_size = ... + + local objects, err = crud.select('customers', conditions, { + first = first, + after = after, + batch_size = batch_size, + }) + return objects, err + ]], {conditions, first, after, batch_size}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, get_by_ids(customers, {3, 4})) + + -- id <= 6 + -- first -2 after 5 (batch_size is 1) + local conditions = { + {'<=', 'id', 6}, + } + local first = -2 + local after = customers[5] + local batch_size = 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, first, after, batch_size = ... + + local objects, err = crud.select('customers', conditions, { + first = first, + after = after, + batch_size = batch_size, + }) + return objects, err + ]], {conditions, first, after, batch_size}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, get_by_ids(customers, {6})) + + -- age <= 66 + -- first -2 after 5 (batch_size is 1) + local conditions = { + {'<=', 'age', 66}, + } + local first = -2 + local after = customers[5] + local batch_size = 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, first, after, batch_size = ... + + local objects, err = crud.select('customers', conditions, { + first = first, + after = after, + batch_size = batch_size, + }) + return objects, err + ]], {conditions, first, after, batch_size}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, get_by_ids(customers, {6})) +end) + add('test_select_all_with_batch_size', function(g) local customers = insert_customers(g, { { @@ -313,13 +489,13 @@ add('test_select_all_with_batch_size', function(g) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, customers) - -- batch size 3 and limit 6 + -- batch size 3 and first 6 local result, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') local result, err = crud.select('customers', nil, { batch_size = 3, - limit = 6, + first = 6, }) return result, err ]]) diff --git a/test/unit/select_plan_test.lua b/test/unit/select_plan_test.lua index 4cd9abd7..d11137a1 100644 --- a/test/unit/select_plan_test.lua +++ b/test/unit/select_plan_test.lua @@ -50,7 +50,7 @@ g.after_all(function() box.space.customers:drop() end) -g.test_scanner_bad_operand_name = function() +g.test_bad_operand_name = function() local plan, err = select_plan.new(box.space.customers, { cond_funcs.gt('non-existent-field-index', 20), }) @@ -60,7 +60,7 @@ g.test_scanner_bad_operand_name = function() t.assert_str_contains(err.err, 'No field or index "non-existent-field-index" found') end -g.test_scanner_indexed_field = function() +g.test_indexed_field = function() -- select by indexed field local conditions = { cond_funcs.gt('age', 20) } @@ -73,13 +73,14 @@ g.test_scanner_indexed_field = function() t.assert_equals(plan.space_name, 'customers') t.assert_equals(plan.index_id, 1) -- age index t.assert_equals(plan.scan_value, {20}) + t.assert_equals(plan.after_tuple, nil) t.assert_equals(plan.scan_condition_num, 1) t.assert_equals(plan.iter, box.index.GT) t.assert_equals(plan.total_tuples_count, nil) t.assert_equals(plan.sharding_key, nil) end -g.test_scanner_non_indexed_field = function() +g.test_non_indexed_field = function() local conditions = { cond_funcs.eq('city', 'Moscow') } local plan, err = select_plan.new(box.space.customers, conditions) @@ -90,13 +91,14 @@ g.test_scanner_non_indexed_field = function() t.assert_equals(plan.space_name, 'customers') t.assert_equals(plan.index_id, 0) -- primary index t.assert_equals(plan.scan_value, {}) + t.assert_equals(plan.after_tuple, nil) t.assert_equals(plan.scan_condition_num, nil) t.assert_equals(plan.iter, box.index.GE) t.assert_equals(plan.total_tuples_count, nil) t.assert_equals(plan.sharding_key, nil) end -g.test_scanner_partial_indexed_field = function() +g.test_partial_indexed_field = function() -- select by first part of the index local conditions = { cond_funcs.gt('name', 'A'), } local plan, err = select_plan.new(box.space.customers, conditions) @@ -108,6 +110,7 @@ g.test_scanner_partial_indexed_field = function() t.assert_equals(plan.space_name, 'customers') t.assert_equals(plan.index_id, 2) -- full_name index t.assert_equals(plan.scan_value, {'A'}) + t.assert_equals(plan.after_tuple, nil) t.assert_equals(plan.scan_condition_num, 1) t.assert_equals(plan.iter, box.index.GT) t.assert_equals(plan.total_tuples_count, nil) @@ -124,6 +127,7 @@ g.test_scanner_partial_indexed_field = function() t.assert_equals(plan.space_name, 'customers') t.assert_equals(plan.index_id, 0) -- primary index t.assert_equals(plan.scan_value, {}) + t.assert_equals(plan.after_tuple, nil) t.assert_equals(plan.scan_condition_num, nil) t.assert_equals(plan.iter, box.index.GE) t.assert_equals(plan.total_tuples_count, nil) @@ -184,3 +188,61 @@ g.test_is_scan_by_full_sharding_key_eq = function() t.assert_equals(plan.total_tuples_count, nil) t.assert_equals(plan.sharding_key, nil) end + +g.test_first = function() + -- positive first + local plan, err = select_plan.new(box.space.customers, nil, { + first = 10, + }) + + t.assert_equals(err, nil) + t.assert_type(plan, 'table') + + t.assert_equals(plan.total_tuples_count, 10) + t.assert_equals(plan.after_tuple, nil) + + -- negative first + + local after_tuple = {777, 1777, 'Leo', 'Tolstoy', 76, 'Tula', false} + + -- select by primary key, no conditions + -- first -10 after_tuple 777 + local plan, err = select_plan.new(box.space.customers, nil, { + first = -10, + after_tuple = after_tuple, + }) + + t.assert_equals(err, nil) + t.assert_type(plan, 'table') + + t.assert_equals(plan.conditions, {}) + t.assert_equals(plan.space_name, 'customers') + t.assert_equals(plan.index_id, 0) -- primary index + t.assert_equals(plan.scan_value, {777}) -- after_tuple id + t.assert_equals(plan.after_tuple, after_tuple) + t.assert_equals(plan.scan_condition_num, nil) + t.assert_equals(plan.iter, box.index.LE) -- inverted iterator + t.assert_equals(plan.total_tuples_count, 10) + t.assert_equals(plan.sharding_key, nil) + + -- select by primary key, lt condition + -- first -10 after_tuple 777 + local conditions = { cond_funcs.lt('age', 90) } + local plan, err = select_plan.new(box.space.customers, conditions, { + first = -10, + after_tuple = after_tuple, + }) + + t.assert_equals(err, nil) + t.assert_type(plan, 'table') + + t.assert_equals(plan.conditions, conditions) + t.assert_equals(plan.space_name, 'customers') + t.assert_equals(plan.index_id, 1) -- primary index + t.assert_equals(plan.scan_value, {76}) -- after_tuple age value + t.assert_equals(plan.after_tuple, after_tuple) -- after_tuple key + t.assert_equals(plan.scan_condition_num, nil) + t.assert_equals(plan.iter, box.index.GT) -- inverted iterator + t.assert_equals(plan.total_tuples_count, 10) + t.assert_equals(plan.sharding_key, nil) +end