diff --git a/CHANGELOG.md b/CHANGELOG.md index bd359762..86d7e1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +* CRUD operations: + * replace + * upsert + ## [0.1.0] - 2020-09-23 ### Added diff --git a/README.md b/README.md index b54090e8..ed0f88f6 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,61 @@ crud.delete('customers', 1) ... ``` +### Replace + +```lua +local object, err = crud.replace(space_name, object, opts) +``` + +where: + +* `space_name` (`string`) - name of the space +* `object` (`table`) - object to insert or replace exist one +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +Returns inserted or replaced object, error. + +**Example:** + +```lua +crud.replace('customers', { + id = 1, name = 'Alice', age = 22, +}) +--- +- bucket_id: 7614 + age: 22 + name: Alice + id: 1 +... +``` + +### Upsert + +```lua +local object, err = crud.upsert(space_name, object, operations, opts) +``` + +where: + +* `space_name` (`string`) - name of the space +* `object` (`table`) - object to insert if there is no existing tuple which matches the key fields +* `operations` (`table`) - update [operations](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/#box-space-update) if there is an existing tuple which matches the key fields of tuple +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +Returns nil, error. + +**Example:** + +```lua +crud.upsert('customers', {id = 1, name = 'Alice', age = 22,}, {{'+', 'age', 1}}) +--- +- nil +... +``` + + ### Select `CRUD` supports multi-conditional selects, treating a cluster as a single space. diff --git a/crud.lua b/crud.lua index 7fe01669..8a0bcc80 100644 --- a/crud.lua +++ b/crud.lua @@ -5,8 +5,10 @@ local registry = require('crud.common.registry') local call = require('crud.common.call') local insert = require('crud.insert') +local replace = require('crud.replace') local get = require('crud.get') local update = require('crud.update') +local upsert = require('crud.upsert') local delete = require('crud.delete') local select = require('crud.select') @@ -30,10 +32,18 @@ crud.insert = insert.call -- @function get crud.get = get.call +-- @refer replace.call +-- @function replace +crud.replace = replace.call + -- @refer update.call -- @function update crud.update = update.call +-- @refer upsert.call +-- @function upsert +crud.upsert = upsert.call + -- @refer delete.call -- @function delete crud.delete = delete.call @@ -57,7 +67,9 @@ function crud.init() call.init() insert.init() get.init() + replace.init() update.init() + upsert.init() delete.init() select.init() end diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 4619fb0a..8c06c8c9 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -3,6 +3,7 @@ local errors = require('errors') local FlattenError = errors.new_class("FlattenError", {capture_stack = false}) local UnflattenError = errors.new_class("UnflattenError", {capture_stack = false}) +local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) local utils = {} @@ -35,7 +36,7 @@ end local system_fields = { bucket_id = true } -function utils.flatten(object, space_format) +function utils.flatten(object, space_format, bucket_id) if object == nil then return nil end local tuple = {} @@ -49,6 +50,10 @@ function utils.flatten(object, space_format) end end + if bucket_id ~= nil and field_format.name == 'bucket_id' then + value = bucket_id + end + tuple[fieldno] = value end @@ -99,4 +104,56 @@ function utils.merge_primary_key_parts(key_parts, pk_parts) return merged_parts end +local __tarantool_supports_fieldpaths +local function tarantool_supports_fieldpaths() + if __tarantool_supports_fieldpaths ~= nil then + return __tarantool_supports_fieldpaths + end + + local major_minor_patch = _G._TARANTOOL:split('-', 1)[1] + local major_minor_patch_parts = major_minor_patch:split('.', 2) + + local major = tonumber(major_minor_patch_parts[1]) + local minor = tonumber(major_minor_patch_parts[2]) + local patch = tonumber(major_minor_patch_parts[3]) + + -- since Tarantool 2.3 + __tarantool_supports_fieldpaths = major >= 2 and (minor > 3 or minor == 3 and patch >= 1) + + return __tarantool_supports_fieldpaths +end + +function utils.convert_operations(user_operations, space_format) + if tarantool_supports_fieldpaths() then + return user_operations + end + + local converted_operations = {} + + for _, operation in ipairs(user_operations) do + if type(operation[2]) == 'string' then + local field_id + for fieldno, field_format in ipairs(space_format) do + if field_format.name == operation[2] then + field_id = fieldno + break + end + end + + if field_id == nil then + return nil, ParseOperationsError:new( + "Space format doesn't contain field named %q", operation[2]) + end + + table.insert(converted_operations, { + operation[1], field_id, operation[3] + }) + else + table.insert(converted_operations, operation) + end + end + + return converted_operations +end + return utils diff --git a/crud/insert.lua b/crud/insert.lua index cab85eb3..c79a00bd 100644 --- a/crud/insert.lua +++ b/crud/insert.lua @@ -62,9 +62,10 @@ function insert.call(space_name, obj, opts) if space == nil then return nil, InsertError:new("Space %q doesn't exists", space_name) end + local space_format = space:format() -- compute default buckect_id - local tuple, err = utils.flatten(obj, space:format(), true) + local tuple, err = utils.flatten(obj, space_format) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end @@ -77,10 +78,7 @@ function insert.call(space_name, obj, opts) return nil, InsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - obj = table.copy(obj) - obj.bucket_id = bucket_id - - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space_format, bucket_id) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end @@ -95,7 +93,7 @@ function insert.call(space_name, obj, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, InsertError:new("Received tuple that doesn't match space format: %s", err) end diff --git a/crud/replace.lua b/crud/replace.lua new file mode 100644 index 00000000..9f830288 --- /dev/null +++ b/crud/replace.lua @@ -0,0 +1,101 @@ +local checks = require('checks') +local errors = require('errors') +local vshard = require('vshard') + +local call = require('crud.common.call') +local registry = require('crud.common.registry') +local utils = require('crud.common.utils') + +require('crud.common.checkers') + +local ReplaceError = errors.new_class('Replace', { capture_stack = false }) + +local replace = {} + +local REPLACE_FUNC_NAME = '__replace' + +local function call_replace_on_storage(space_name, tuple) + checks('string', 'table') + + local space = box.space[space_name] + if space == nil then + return nil, ReplaceError:new("Space %q doesn't exists", space_name) + end + + return space:replace(tuple) +end + +function replace.init() + registry.add({ + [REPLACE_FUNC_NAME] = call_replace_on_storage, + }) +end + +--- Insert or replace a tuple in the specifed space +-- +-- @function call +-- +-- @param string space_name +-- A space name +-- +-- @param table obj +-- Tuple object (according to space format) +-- +-- @tparam ?number opts.timeout +-- Function call timeout +-- +-- @return[1] object +-- @treturn[2] nil +-- @treturn[2] table Error description +-- +function replace.call(space_name, obj, opts) + checks('string', 'table', { + timeout = '?number', + }) + + opts = opts or {} + + local space = utils.get_space(space_name, vshard.router.routeall()) + if space == nil then + return nil, ReplaceError:new("Space %q doesn't exists", space_name) + end + + local space_format = space:format() + -- compute default buckect_id + local tuple, err = utils.flatten(obj, space_format) + if err ~= nil then + return nil, ReplaceError:new("Object is specified in bad format: %s", err) + end + + local key = utils.extract_key(tuple, space.index[0].parts) + + local bucket_id = vshard.router.bucket_id_strcrc32(key) + local replicaset, err = vshard.router.route(bucket_id) + if replicaset == nil then + return nil, ReplaceError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) + end + + local tuple, err = utils.flatten(obj, space_format, bucket_id) + if err ~= nil then + return nil, ReplaceError:new("Object is specified in bad format: %s", err) + end + + local results, err = call.rw(REPLACE_FUNC_NAME, {space_name, tuple}, { + replicasets = {replicaset}, + timeout = opts.timeout, + }) + + if err ~= nil then + return nil, ReplaceError:new("Failed to replace: %s", err) + end + + local tuple = results[replicaset.uuid] + local object, err = utils.unflatten(tuple, space_format) + if err ~= nil then + return nil, ReplaceError:new("Received tuple that doesn't match space format: %s", err) + end + + return object +end + +return replace diff --git a/crud/update.lua b/crud/update.lua index cced851d..9dded443 100644 --- a/crud/update.lua +++ b/crud/update.lua @@ -9,7 +9,6 @@ local utils = require('crud.common.utils') require('crud.common.checkers') local UpdateError = errors.new_class('Update', {capture_stack = false}) -local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) local update = {} @@ -33,58 +32,6 @@ function update.init() }) end -local __tarantool_supports_fieldpaths -local function tarantool_supports_fieldpaths() - if __tarantool_supports_fieldpaths ~= nil then - return __tarantool_supports_fieldpaths - end - - local major_minor_patch = _G._TARANTOOL:split('-', 1)[1] - local major_minor_patch_parts = major_minor_patch:split('.', 2) - - local major = tonumber(major_minor_patch_parts[1]) - local minor = tonumber(major_minor_patch_parts[2]) - local patch = tonumber(major_minor_patch_parts[3]) - - -- since Tarantool 2.3 - __tarantool_supports_fieldpaths = major >= 2 and (minor > 3 or minor == 3 and patch >= 1) - - return __tarantool_supports_fieldpaths -end - -local function convert_operations(user_operations, space_format) - if tarantool_supports_fieldpaths() then - return user_operations - end - - local converted_operations = {} - - for _, operation in ipairs(user_operations) do - if type(operation[2]) == 'string' then - local field_id - for fieldno, field_format in ipairs(space_format) do - if field_format.name == operation[2] then - field_id = fieldno - break - end - end - - if field_id == nil then - return nil, ParseOperationsError:new( - "Space format doesn't contain field named %q", operation[2]) - end - - table.insert(converted_operations, { - operation[1], field_id, operation[3] - }) - else - table.insert(converted_operations, operation) - end - end - - return converted_operations -end - --- Updates tuple in the specifed space -- -- @function call @@ -95,7 +42,7 @@ end -- @param key -- Primary key value -- --- @param table operations +-- @param table user_operations -- Operations to be performed. -- See `spaceect:update` operations in Tarantool doc -- @@ -117,12 +64,13 @@ function update.call(space_name, key, user_operations, opts) if space == nil then return nil, UpdateError:new("Space %q doesn't exists", space_name) end + local space_format = space:format() if box.tuple.is(key) then key = key:totable() end - local operations, err = convert_operations(user_operations, space:format()) + local operations, err = utils.convert_operations(user_operations, space_format) if err ~= nil then return nil, UpdateError:new("Wrong operations are specified: %s", err) end @@ -143,7 +91,7 @@ function update.call(space_name, key, user_operations, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, UpdateError:new("Received tuple that doesn't match space format: %s", err) end diff --git a/crud/upsert.lua b/crud/upsert.lua new file mode 100644 index 00000000..bddaf663 --- /dev/null +++ b/crud/upsert.lua @@ -0,0 +1,105 @@ +local checks = require('checks') +local errors = require('errors') +local vshard = require('vshard') + +local call = require('crud.common.call') +local registry = require('crud.common.registry') +local utils = require('crud.common.utils') + +require('crud.common.checkers') + +local UpsertError = errors.new_class('UpsertError', { capture_stack = false}) + +local upsert = {} + +local UPSERT_FUNC_NAME = '__upsert' + +local function call_upsert_on_storage(space_name, tuple, operations) + checks('string', 'table', 'update_operations') + + local space = box.space[space_name] + if space == nil then + return nil, UpsertError:new("Space %q doesn't exists", space_name) + end + + return space:upsert(tuple, operations) +end + +function upsert.init() + registry.add({ + [UPSERT_FUNC_NAME] = call_upsert_on_storage, + }) +end + +--- Update or insert a tuple in the specified space +-- +-- @function call +-- +-- @param string space_name +-- A space name +-- +-- @param table obj +-- Tuple object (according to space format) +-- +-- @param table user_operations +-- user_operations to be performed. +-- See `space_object:update` operations in Tarantool doc +-- +-- @tparam ?number opts.timeout +-- Function call timeout +-- +-- @return[1] object +-- @treturn[2] nil +-- @treturn[2] table Error description +-- +function upsert.call(space_name, obj, user_operations, opts) + checks('string', '?', 'update_operations', { + timeout = '?number', + }) + + opts = opts or {} + + local space = utils.get_space(space_name, vshard.router.routeall()) + if space == nil then + return nil, UpsertError:new("Space %q doesn't exists", space_name) + end + + local space_format = space:format() + local operations, err = utils.convert_operations(user_operations, space_format) + if err ~= nil then + return nil, UpsertError:new("Wrong operations are specified: %s", err) + end + + -- compute default buckect_id + local tuple, err = utils.flatten(obj, space_format) + if err ~= nil then + return nil, UpsertError:new("Object is specified in bad format: %s", err) + end + + local key = utils.extract_key(tuple, space.index[0].parts) + + local bucket_id = vshard.router.bucket_id_strcrc32(key) + local replicaset, err = vshard.router.route(bucket_id) + if replicaset == nil then + return nil, UpsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) + end + + local tuple, err = utils.flatten(obj, space_format, bucket_id) + if err ~= nil then + return nil, UpsertError:new("Object is specified in bad format: %s", err) + end + + local results, err = call.rw(UPSERT_FUNC_NAME, {space_name, tuple, operations}, { + replicasets = {replicaset}, + timeout = opts.timeout, + }) + + if err ~= nil then + return nil, UpsertError:new("Failed to upsert: %s", err) + end + + --upsert always return nil + return results[replicaset.uuid] +end + +return upsert diff --git a/test/entrypoint/srv_simple_operations.lua b/test/entrypoint/srv_simple_operations.lua index be2f362a..16c82494 100755 --- a/test/entrypoint/srv_simple_operations.lua +++ b/test/entrypoint/srv_simple_operations.lua @@ -11,6 +11,7 @@ package.preload['customers-storage'] = function() return { role_name = 'customers-storage', init = function() + local engine = os.getenv('ENGINE') or 'memtx' local customers_space = box.schema.space.create('customers', { format = { {name = 'id', type = 'unsigned'}, @@ -19,6 +20,7 @@ package.preload['customers-storage'] = function() {name = 'age', type = 'number'}, }, if_not_exists = true, + engine = engine, }) customers_space:create_index('id', { parts = { {field = 'id'} }, diff --git a/test/integration/simple_operations_test.lua b/test/integration/simple_operations_test.lua index 8291fbaf..1506ad06 100644 --- a/test/integration/simple_operations_test.lua +++ b/test/integration/simple_operations_test.lua @@ -1,13 +1,14 @@ local fio = require('fio') local t = require('luatest') -local g = t.group('simple_operations') +local g_memtx = t.group('simple_operations_memtx') +local g_vinyl = t.group('simple_operations_vinyl') local helpers = require('test.helper') math.randomseed(os.time()) -g.before_all = function() +local function before_all(g, engine) g.cluster = helpers.Cluster:new({ datadir = fio.tempdir(), server_command = helpers.entrypoint('srv_simple_operations'), @@ -40,16 +41,26 @@ g.before_all = function() }, } }, + env = { + ['ENGINE'] = engine, + }, }) + g.engine = engine g.cluster:start() end -g.after_all = function() +g_memtx.before_all = function() before_all(g_memtx, 'memtx') end +g_vinyl.before_all = function() before_all(g_vinyl, 'vinyl') end + +local function after_all(g) g.cluster:stop() fio.rmtree(g.cluster.datadir) end -g.before_each(function() +g_memtx.after_all = function() after_all(g_memtx) end +g_vinyl.after_all = function() after_all(g_vinyl) end + +local function before_each(g) for _, server in ipairs(g.cluster.servers) do server.net_box:eval([[ local space = box.space.customers @@ -58,11 +69,20 @@ g.before_each(function() end ]]) end -end) +end + +g_memtx.before_each(function() before_each(g_memtx) end) +g_vinyl.before_each(function() before_each(g_vinyl) end) -g.test_non_existent_space = function() +local function add(name, fn) + g_memtx[name] = fn + g_vinyl[name] = fn +end + +add('test_non_existent_space', function(g) -- insert local obj, err = g.cluster.main_server.net_box:eval([[ + local space_name = ... local crud = require('crud') return crud.insert('non_existent_space', {id = 0, name = 'Fedor', age = 59}) ]]) @@ -96,9 +116,27 @@ g.test_non_existent_space = function() t.assert_equals(obj, nil) t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') -end -g.test_insert_get = function() + -- replace + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('non_existent_space', {id = 0, name = 'Fedor', age = 59}) + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') + + -- upsert + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('non_existent_space', {id = 0, name = 'Fedor', age = 59}, {{'+', 'age', 1}}) + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') +end) + +add('test_insert_get', function(g) -- insert local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -147,9 +185,9 @@ g.test_insert_get = function() t.assert_equals(err, nil) t.assert_equals(obj, nil) -end +end) -g.test_update = function() +add('test_update', function(g) -- insert tuple local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -215,9 +253,9 @@ g.test_update = function() t.assert_equals(err, nil) t.assert_covers(obj, {id = 22, name = 'Leo', age = 72}) t.assert(type(obj.bucket_id) == 'number') -end +end) -g.test_delete = function() +add('test_delete', function(g) -- insert tuple local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -235,8 +273,12 @@ g.test_delete = function() ]]) t.assert_equals(err, nil) - t.assert_covers(obj, {id = 33, name = 'Mayakovsky', age = 36}) - t.assert(type(obj.bucket_id) == 'number') + if g.engine == 'memtx' then + t.assert_covers(obj, {id = 33, name = 'Mayakovsky', age = 36}) + t.assert(type(obj.bucket_id) == 'number') + else + t.assert_equals(obj, nil) + end -- get local obj, err = g.cluster.main_server.net_box:eval([[ @@ -256,5 +298,79 @@ g.test_delete = function() t.assert_equals(obj, nil) t.assert_str_contains(err.err, "Failed for %w+%-0000%-0000%-0000%-000000000000", true) t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type:") -end +end) + +add('test_replace', function(g) + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 44) + ]]) + t.assert_equals(err, nil) + t.assert_equals(obj, nil) + + -- insert tuple + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('customers', {id = 44, name = 'John Doe', age = 25}) + ]]) + + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 44, name = 'John Doe', age = 25}) + t.assert(type(obj.bucket_id) == 'number') + + -- replace tuple + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('customers', {id = 44, name = 'Jane Doe', age = 18}) + ]]) + + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 44, name = 'Jane Doe', age = 18}) + t.assert(type(obj.bucket_id) == 'number') +end) + +add('test_upsert', function(g) + -- upsert tuple not exist + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('customers', {id = 66, name = 'Jack Sparrow', age = 25}, { + {'+', 'age', 25}, + {'=', 'name', 'Leo Tolstoy'} + }) + ]]) + + t.assert_equals(obj, nil) + t.assert_equals(err, nil) + + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 66) + ]]) + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 66, name = 'Jack Sparrow', age = 25}) + t.assert(type(obj.bucket_id) == 'number') + + -- upsert same query second time tuple exist + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('customers', {id = 66, name = 'Jack Sparrow', age = 25}, { + {'+', 'age', 25}, + {'=', 'name', 'Leo Tolstoy'} + }) + ]]) + + t.assert_equals(obj, nil) + t.assert_equals(err, nil) + + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 66) + ]]) + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 66, name = 'Leo Tolstoy', age = 50}) + t.assert(type(obj.bucket_id) == 'number') +end) diff --git a/test/unit/serialization_test.lua b/test/unit/serialization_test.lua index 9585d2b7..92dd86fa 100644 --- a/test/unit/serialization_test.lua +++ b/test/unit/serialization_test.lua @@ -42,6 +42,18 @@ g.test_flatten = function() t.assert(err == nil) t.assert_equals(tuple, {1, 1024, 'Marilyn', 50}) + + -- set bucket_id + local object = { + id = 1, + name = 'Marilyn', + age = 50, + } + + local tuple, err = utils.flatten(object, space_format, 1025) + t.assert(err == nil) + t.assert_equals(tuple, {1, 1025, 'Marilyn', 50}) + -- non-nullable field name is nil local object = { id = 1, @@ -63,7 +75,7 @@ g.test_flatten = function() age = 50, } - local tuple, err = utils.flatten(object, space_format, true) + local tuple, err = utils.flatten(object, space_format) t.assert(err == nil) t.assert_equals(tuple, {1, nil, 'Marilyn', 50})