From 7f6871901c113804fb8ceca5001402d16b1a0e61 Mon Sep 17 00:00:00 2001 From: Sergey Volgin Date: Wed, 23 Sep 2020 21:32:15 +0400 Subject: [PATCH 1/6] Support replace and upsert operations --- README.md | 55 ++++++++++ crud.lua | 12 +++ crud/common/utils.lua | 53 +++++++++ crud/replace.lua | 103 ++++++++++++++++++ crud/update.lua | 57 +--------- crud/upsert.lua | 112 ++++++++++++++++++++ test/integration/simple_operations_test.lua | 74 +++++++++++++ 7 files changed, 411 insertions(+), 55 deletions(-) create mode 100644 crud/replace.lua create mode 100644 crud/upsert.lua diff --git a/README.md b/README.md index b54090e8..ed0f88f6 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,61 @@ crud.delete('customers', 1) ... ``` +### Replace + +```lua +local object, err = crud.replace(space_name, object, opts) +``` + +where: + +* `space_name` (`string`) - name of the space +* `object` (`table`) - object to insert or replace exist one +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +Returns inserted or replaced object, error. + +**Example:** + +```lua +crud.replace('customers', { + id = 1, name = 'Alice', age = 22, +}) +--- +- bucket_id: 7614 + age: 22 + name: Alice + id: 1 +... +``` + +### Upsert + +```lua +local object, err = crud.upsert(space_name, object, operations, opts) +``` + +where: + +* `space_name` (`string`) - name of the space +* `object` (`table`) - object to insert if there is no existing tuple which matches the key fields +* `operations` (`table`) - update [operations](https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/#box-space-update) if there is an existing tuple which matches the key fields of tuple +* `opts`: + * `timeout` (`?number`) - `vshard.call` timeout (in seconds) + +Returns nil, error. + +**Example:** + +```lua +crud.upsert('customers', {id = 1, name = 'Alice', age = 22,}, {{'+', 'age', 1}}) +--- +- nil +... +``` + + ### Select `CRUD` supports multi-conditional selects, treating a cluster as a single space. diff --git a/crud.lua b/crud.lua index 7fe01669..8a0bcc80 100644 --- a/crud.lua +++ b/crud.lua @@ -5,8 +5,10 @@ local registry = require('crud.common.registry') local call = require('crud.common.call') local insert = require('crud.insert') +local replace = require('crud.replace') local get = require('crud.get') local update = require('crud.update') +local upsert = require('crud.upsert') local delete = require('crud.delete') local select = require('crud.select') @@ -30,10 +32,18 @@ crud.insert = insert.call -- @function get crud.get = get.call +-- @refer replace.call +-- @function replace +crud.replace = replace.call + -- @refer update.call -- @function update crud.update = update.call +-- @refer upsert.call +-- @function upsert +crud.upsert = upsert.call + -- @refer delete.call -- @function delete crud.delete = delete.call @@ -57,7 +67,9 @@ function crud.init() call.init() insert.init() get.init() + replace.init() update.init() + upsert.init() delete.init() select.init() end diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 4619fb0a..d91baa16 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -3,6 +3,7 @@ local errors = require('errors') local FlattenError = errors.new_class("FlattenError", {capture_stack = false}) local UnflattenError = errors.new_class("UnflattenError", {capture_stack = false}) +local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) local utils = {} @@ -99,4 +100,56 @@ function utils.merge_primary_key_parts(key_parts, pk_parts) return merged_parts end +local __tarantool_supports_fieldpaths +local function tarantool_supports_fieldpaths() + if __tarantool_supports_fieldpaths ~= nil then + return __tarantool_supports_fieldpaths + end + + local major_minor_patch = _G._TARANTOOL:split('-', 1)[1] + local major_minor_patch_parts = major_minor_patch:split('.', 2) + + local major = tonumber(major_minor_patch_parts[1]) + local minor = tonumber(major_minor_patch_parts[2]) + local patch = tonumber(major_minor_patch_parts[3]) + + -- since Tarantool 2.3 + __tarantool_supports_fieldpaths = major >= 2 and (minor > 3 or minor == 3 and patch >= 1) + + return __tarantool_supports_fieldpaths +end + +function utils.convert_operations(user_operations, space_format) + if tarantool_supports_fieldpaths() then + return user_operations + end + + local converted_operations = {} + + for _, operation in ipairs(user_operations) do + if type(operation[2]) == 'string' then + local field_id + for fieldno, field_format in ipairs(space_format) do + if field_format.name == operation[2] then + field_id = fieldno + break + end + end + + if field_id == nil then + return nil, ParseOperationsError:new( + "Space format doesn't contain field named %q", operation[2]) + end + + table.insert(converted_operations, { + operation[1], field_id, operation[3] + }) + else + table.insert(converted_operations, operation) + end + end + + return converted_operations +end + return utils diff --git a/crud/replace.lua b/crud/replace.lua new file mode 100644 index 00000000..88040357 --- /dev/null +++ b/crud/replace.lua @@ -0,0 +1,103 @@ +local checks = require('checks') +local errors = require('errors') +local vshard = require('vshard') + +local call = require('crud.common.call') +local registry = require('crud.common.registry') +local utils = require('crud.common.utils') + +require('crud.common.checkers') + +local ReplaceError = errors.new_class('Replace', { capture_stack = false }) + +local replace = {} + +local REPLACE_FUNC_NAME = '__replace' + +local function call_replace_on_storage(space_name, tuple) + checks('string', 'table') + + local space = box.space[space_name] + if space == nil then + return nil, ReplaceError:new("Space %q doesn't exists", space_name) + end + + return space:replace(tuple) +end + +function replace.init() + registry.add({ + [REPLACE_FUNC_NAME] = call_replace_on_storage, + }) +end + +--- Insert or replace a tuple in the specifed space +-- +-- @function call +-- +-- @param string space_name +-- A space name +-- +-- @param table obj +-- Tuple object (according to space format) +-- +-- @tparam ?number opts.timeout +-- Function call timeout +-- +-- @return[1] object +-- @treturn[2] nil +-- @treturn[2] table Error description +-- +function replace.call(space_name, obj, opts) + checks('string', 'table', { + timeout = '?number', + }) + + opts = opts or {} + + local space = utils.get_space(space_name, vshard.router.routeall()) + if space == nil then + return nil, ReplaceError:new("Space %q doesn't exists", space_name) + end + + -- compute default buckect_id + local tuple, err = utils.flatten(obj, space:format()) + if err ~= nil then + return nil, ReplaceError:new("Object is specified in bad format: %s", err) + end + + local key = utils.extract_key(tuple, space.index[0].parts) + + local bucket_id = vshard.router.bucket_id_strcrc32(key) + local replicaset, err = vshard.router.route(bucket_id) + if replicaset == nil then + return nil, ReplaceError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) + end + + obj = table.copy(obj) + obj.bucket_id = bucket_id + + local tuple, err = utils.flatten(obj, space:format()) + if err ~= nil then + return nil, ReplaceError:new("Object is specified in bad format: %s", err) + end + + local results, err = call.rw(REPLACE_FUNC_NAME, {space_name, tuple}, { + replicasets = {replicaset}, + timeout = opts.timeout, + }) + + if err ~= nil then + return nil, ReplaceError:new("Failed to replace: %s", err) + end + + local tuple = results[replicaset.uuid] + local object, err = utils.unflatten(tuple, space:format()) + if err ~= nil then + return nil, ReplaceError:new("Received tuple that doesn't match space format: %s", err) + end + + return object +end + +return replace diff --git a/crud/update.lua b/crud/update.lua index cced851d..59c611fd 100644 --- a/crud/update.lua +++ b/crud/update.lua @@ -9,7 +9,6 @@ local utils = require('crud.common.utils') require('crud.common.checkers') local UpdateError = errors.new_class('Update', {capture_stack = false}) -local ParseOperationsError = errors.new_class('ParseOperationsError', {capture_stack = false}) local update = {} @@ -33,58 +32,6 @@ function update.init() }) end -local __tarantool_supports_fieldpaths -local function tarantool_supports_fieldpaths() - if __tarantool_supports_fieldpaths ~= nil then - return __tarantool_supports_fieldpaths - end - - local major_minor_patch = _G._TARANTOOL:split('-', 1)[1] - local major_minor_patch_parts = major_minor_patch:split('.', 2) - - local major = tonumber(major_minor_patch_parts[1]) - local minor = tonumber(major_minor_patch_parts[2]) - local patch = tonumber(major_minor_patch_parts[3]) - - -- since Tarantool 2.3 - __tarantool_supports_fieldpaths = major >= 2 and (minor > 3 or minor == 3 and patch >= 1) - - return __tarantool_supports_fieldpaths -end - -local function convert_operations(user_operations, space_format) - if tarantool_supports_fieldpaths() then - return user_operations - end - - local converted_operations = {} - - for _, operation in ipairs(user_operations) do - if type(operation[2]) == 'string' then - local field_id - for fieldno, field_format in ipairs(space_format) do - if field_format.name == operation[2] then - field_id = fieldno - break - end - end - - if field_id == nil then - return nil, ParseOperationsError:new( - "Space format doesn't contain field named %q", operation[2]) - end - - table.insert(converted_operations, { - operation[1], field_id, operation[3] - }) - else - table.insert(converted_operations, operation) - end - end - - return converted_operations -end - --- Updates tuple in the specifed space -- -- @function call @@ -95,7 +42,7 @@ end -- @param key -- Primary key value -- --- @param table operations +-- @param table user_operations -- Operations to be performed. -- See `spaceect:update` operations in Tarantool doc -- @@ -122,7 +69,7 @@ function update.call(space_name, key, user_operations, opts) key = key:totable() end - local operations, err = convert_operations(user_operations, space:format()) + local operations, err = utils.convert_operations(user_operations, space:format()) if err ~= nil then return nil, UpdateError:new("Wrong operations are specified: %s", err) end diff --git a/crud/upsert.lua b/crud/upsert.lua new file mode 100644 index 00000000..298ab4ee --- /dev/null +++ b/crud/upsert.lua @@ -0,0 +1,112 @@ +local checks = require('checks') +local errors = require('errors') +local vshard = require('vshard') + +local call = require('crud.common.call') +local registry = require('crud.common.registry') +local utils = require('crud.common.utils') + +require('crud.common.checkers') + +local UpsertError = errors.new_class('UpsertError', { capture_stack = false}) + +local upsert = {} + +local UPSERT_FUNC_NAME = '__upsert' + +local function call_upsert_on_storage(space_name, tuple, operations) + checks('string', 'table', 'update_operations') + + local space = box.space[space_name] + if space == nil then + return nil, UpdateError:new("Space %q doesn't exists", space_name) + end + + return space:upsert(tuple, operations) +end + +function upsert.init() + registry.add({ + [UPSERT_FUNC_NAME] = call_upsert_on_storage, + }) +end + +--- Update or insert a tuple in the specified space +-- +-- @function call +-- +-- @param string space_name +-- A space name +-- +-- @param table obj +-- Tuple object (according to space format) +-- +-- @param table user_operations +-- user_operations to be performed. +-- See `space_object:update` operations in Tarantool doc +-- +-- @tparam ?number opts.timeout +-- Function call timeout +-- +-- @return[1] object +-- @treturn[2] nil +-- @treturn[2] table Error description +-- +function upsert.call(space_name, obj, user_operations, opts) + checks('string', '?', 'update_operations', { + timeout = '?number', + }) + + opts = opts or {} + + local space = utils.get_space(space_name, vshard.router.routeall()) + if space == nil then + return nil, UpsertError:new("Space %q doesn't exists", space_name) + end + + local operations, err = utils.convert_operations(user_operations, space:format()) + if err ~= nil then + return nil, UpsertError:new("Wrong operations are specified: %s", err) + end + + -- compute default buckect_id + local tuple, err = utils.flatten(obj, space:format()) + if err ~= nil then + return nil, UpsertError:new("Object is specified in bad format: %s", err) + end + + local key = utils.extract_key(tuple, space.index[0].parts) + + local bucket_id = vshard.router.bucket_id_strcrc32(key) + local replicaset, err = vshard.router.route(bucket_id) + if replicaset == nil then + return nil, UpsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) + end + + obj = table.copy(obj) + obj.bucket_id = bucket_id + + local tuple, err = utils.flatten(obj, space:format()) + if err ~= nil then + return nil, UpsertError:new("Object is specified in bad format: %s", err) + end + + local results, err = call.rw(UPSERT_FUNC_NAME, {space_name, tuple, operations}, { + replicasets = {replicaset}, + timeout = opts.timeout, + }) + + if err ~= nil then + return nil, UpsertError:new("Failed to upsert: %s", err) + end + + local tuple = results[replicaset.uuid] + local object, err = utils.unflatten(tuple, space:format()) + if err ~= nil then + return nil, UpsertError:new("Received tuple that doesn't match space format: %s", err) + end + + return object +end + +return upsert diff --git a/test/integration/simple_operations_test.lua b/test/integration/simple_operations_test.lua index 8291fbaf..674b6e60 100644 --- a/test/integration/simple_operations_test.lua +++ b/test/integration/simple_operations_test.lua @@ -258,3 +258,77 @@ g.test_delete = function() t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type:") end +g.test_replace = function() + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 44) + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj, nil) + + -- insert tuple + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('customers', {id = 44, name = 'John Doe', age = 25}) + ]]) + + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 44, name = 'John Doe', age = 25}) + t.assert(type(obj.bucket_id) == 'number') + + -- replace tuple + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('customers', {id = 44, name = 'Jane Doe', age = 18}) + ]]) + + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 44, name = 'Jane Doe', age = 18}) + t.assert(type(obj.bucket_id) == 'number') +end + +g.test_upsert = function() + -- upsert tuple not exist + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('customers', {id = 66, name = 'Jack Sparrow', age = 25}, { + {'+', 'age', 25}, + {'=', 'name', 'Leo Tolstoy'} + }) + ]]) + + t.assert_equals(obj, nil) + t.assert_equals(err, nil) + + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 66) + ]]) + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 66, name = 'Jack Sparrow', age = 25}) + t.assert(type(obj.bucket_id) == 'number') + + -- upsert same query second time tuple exist + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('customers', {id = 66, name = 'Jack Sparrow', age = 25}, { + {'+', 'age', 25}, + {'=', 'name', 'Leo Tolstoy'} + }) + ]]) + + t.assert_equals(obj, nil) + t.assert_equals(err, nil) + + -- get + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.get('customers', 66) + ]]) + t.assert_equals(err, nil) + t.assert_covers(obj, {id = 66, name = 'Leo Tolstoy', age = 50}) + t.assert(type(obj.bucket_id) == 'number') +end From 34c019fd9b390c4beb7e053d4b27984ee155d700 Mon Sep 17 00:00:00 2001 From: Sergey Volgin Date: Thu, 24 Sep 2020 15:54:23 +0400 Subject: [PATCH 2/6] * Small perfomance fix * add vinil engine for simple operation test --- crud/common/utils.lua | 6 +- crud/insert.lua | 7 +- crud/replace.lua | 8 +-- crud/upsert.lua | 5 +- test/entrypoint/srv_simple_operations.lua | 2 + test/integration/simple_operations_test.lua | 79 ++++++++++++++++----- test/unit/serialization_test.lua | 14 +++- 7 files changed, 87 insertions(+), 34 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index d91baa16..6c89cdac 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -36,7 +36,7 @@ end local system_fields = { bucket_id = true } -function utils.flatten(object, space_format) +function utils.flatten(object, space_format, bucket_id) if object == nil then return nil end local tuple = {} @@ -48,6 +48,10 @@ function utils.flatten(object, space_format) if not field_format.is_nullable and value == nil then return nil, FlattenError:new("Field %q isn't nullable", field_format.name) end + else + if bucket_id ~= nil then + value = bucket_id + end end tuple[fieldno] = value diff --git a/crud/insert.lua b/crud/insert.lua index cab85eb3..673f87de 100644 --- a/crud/insert.lua +++ b/crud/insert.lua @@ -64,7 +64,7 @@ function insert.call(space_name, obj, opts) end -- compute default buckect_id - local tuple, err = utils.flatten(obj, space:format(), true) + local tuple, err = utils.flatten(obj, space:format()) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end @@ -77,10 +77,7 @@ function insert.call(space_name, obj, opts) return nil, InsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - obj = table.copy(obj) - obj.bucket_id = bucket_id - - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space:format(), bucket_id) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end diff --git a/crud/replace.lua b/crud/replace.lua index 88040357..80b4a1aa 100644 --- a/crud/replace.lua +++ b/crud/replace.lua @@ -60,8 +60,9 @@ function replace.call(space_name, obj, opts) return nil, ReplaceError:new("Space %q doesn't exists", space_name) end + local space_format = space:format() -- compute default buckect_id - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space_format) if err ~= nil then return nil, ReplaceError:new("Object is specified in bad format: %s", err) end @@ -74,10 +75,7 @@ function replace.call(space_name, obj, opts) return nil, ReplaceError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - obj = table.copy(obj) - obj.bucket_id = bucket_id - - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space_format, bucket_id) if err ~= nil then return nil, ReplaceError:new("Object is specified in bad format: %s", err) end diff --git a/crud/upsert.lua b/crud/upsert.lua index 298ab4ee..46bc59b3 100644 --- a/crud/upsert.lua +++ b/crud/upsert.lua @@ -83,10 +83,7 @@ function upsert.call(space_name, obj, user_operations, opts) return nil, UpsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - obj = table.copy(obj) - obj.bucket_id = bucket_id - - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space:format(), bucket_id) if err ~= nil then return nil, UpsertError:new("Object is specified in bad format: %s", err) end diff --git a/test/entrypoint/srv_simple_operations.lua b/test/entrypoint/srv_simple_operations.lua index be2f362a..16c82494 100755 --- a/test/entrypoint/srv_simple_operations.lua +++ b/test/entrypoint/srv_simple_operations.lua @@ -11,6 +11,7 @@ package.preload['customers-storage'] = function() return { role_name = 'customers-storage', init = function() + local engine = os.getenv('ENGINE') or 'memtx' local customers_space = box.schema.space.create('customers', { format = { {name = 'id', type = 'unsigned'}, @@ -19,6 +20,7 @@ package.preload['customers-storage'] = function() {name = 'age', type = 'number'}, }, if_not_exists = true, + engine = engine, }) customers_space:create_index('id', { parts = { {field = 'id'} }, diff --git a/test/integration/simple_operations_test.lua b/test/integration/simple_operations_test.lua index 674b6e60..ff5a7805 100644 --- a/test/integration/simple_operations_test.lua +++ b/test/integration/simple_operations_test.lua @@ -1,13 +1,15 @@ local fio = require('fio') local t = require('luatest') +local g_memtx = t.group('simple_operations_memtx') +local g_vinyl = t.group('simple_operations_vinyl') local g = t.group('simple_operations') local helpers = require('test.helper') math.randomseed(os.time()) -g.before_all = function() +local function before_all(g, engine) g.cluster = helpers.Cluster:new({ datadir = fio.tempdir(), server_command = helpers.entrypoint('srv_simple_operations'), @@ -40,16 +42,26 @@ g.before_all = function() }, } }, + env = { + ['ENGINE'] = engine, + }, }) + g.engine = engine g.cluster:start() end -g.after_all = function() +g_memtx.before_all = function() before_all(g_memtx, 'memtx') end +g_vinyl.before_all = function() before_all(g_vinyl, 'vinyl') end + +local function after_all(g) g.cluster:stop() fio.rmtree(g.cluster.datadir) end -g.before_each(function() +g_memtx.after_all = function() after_all(g_memtx) end +g_vinyl.after_all = function() after_all(g_vinyl) end + +local function before_each(g) for _, server in ipairs(g.cluster.servers) do server.net_box:eval([[ local space = box.space.customers @@ -58,11 +70,20 @@ g.before_each(function() end ]]) end -end) +end -g.test_non_existent_space = function() +g_memtx.before_each(function() before_each(g_memtx) end) +g_vinyl.before_each(function() before_each(g_vinyl) end) + +local function add(name, fn) + g_memtx[name] = fn + g_vinyl[name] = fn +end + +add('test_non_existent_space', function(g) -- insert local obj, err = g.cluster.main_server.net_box:eval([[ + local space_name = ... local crud = require('crud') return crud.insert('non_existent_space', {id = 0, name = 'Fedor', age = 59}) ]]) @@ -96,9 +117,27 @@ g.test_non_existent_space = function() t.assert_equals(obj, nil) t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') -end -g.test_insert_get = function() + -- replace + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.replace('non_existent_space', {id = 0, name = 'Fedor', age = 59}) + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') + + -- upsert + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + return crud.upsert('non_existent_space', {id = 0, name = 'Fedor', age = 59}, {{'+', 'age', 1}}) + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, 'Space "non_existent_space" doesn\'t exists') +end) + +add('test_insert_get', function(g) -- insert local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -147,9 +186,9 @@ g.test_insert_get = function() t.assert_equals(err, nil) t.assert_equals(obj, nil) -end +end) -g.test_update = function() +add('test_update', function(g) -- insert tuple local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -215,9 +254,9 @@ g.test_update = function() t.assert_equals(err, nil) t.assert_covers(obj, {id = 22, name = 'Leo', age = 72}) t.assert(type(obj.bucket_id) == 'number') -end +end) -g.test_delete = function() +add('test_delete', function(g) -- insert tuple local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -235,8 +274,12 @@ g.test_delete = function() ]]) t.assert_equals(err, nil) - t.assert_covers(obj, {id = 33, name = 'Mayakovsky', age = 36}) - t.assert(type(obj.bucket_id) == 'number') + if g.engine == 'memtx' then + t.assert_covers(obj, {id = 33, name = 'Mayakovsky', age = 36}) + t.assert(type(obj.bucket_id) == 'number') + else + t.assert_equals(obj, nil) + end -- get local obj, err = g.cluster.main_server.net_box:eval([[ @@ -256,9 +299,9 @@ g.test_delete = function() t.assert_equals(obj, nil) t.assert_str_contains(err.err, "Failed for %w+%-0000%-0000%-0000%-000000000000", true) t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type:") -end +end) -g.test_replace = function() +add('test_replace', function(g) -- get local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -287,9 +330,9 @@ g.test_replace = function() t.assert_equals(err, nil) t.assert_covers(obj, {id = 44, name = 'Jane Doe', age = 18}) t.assert(type(obj.bucket_id) == 'number') -end +end) -g.test_upsert = function() +add('test_upsert', function(g) -- upsert tuple not exist local obj, err = g.cluster.main_server.net_box:eval([[ local crud = require('crud') @@ -331,4 +374,4 @@ g.test_upsert = function() t.assert_equals(err, nil) t.assert_covers(obj, {id = 66, name = 'Leo Tolstoy', age = 50}) t.assert(type(obj.bucket_id) == 'number') -end +end) diff --git a/test/unit/serialization_test.lua b/test/unit/serialization_test.lua index 9585d2b7..92dd86fa 100644 --- a/test/unit/serialization_test.lua +++ b/test/unit/serialization_test.lua @@ -42,6 +42,18 @@ g.test_flatten = function() t.assert(err == nil) t.assert_equals(tuple, {1, 1024, 'Marilyn', 50}) + + -- set bucket_id + local object = { + id = 1, + name = 'Marilyn', + age = 50, + } + + local tuple, err = utils.flatten(object, space_format, 1025) + t.assert(err == nil) + t.assert_equals(tuple, {1, 1025, 'Marilyn', 50}) + -- non-nullable field name is nil local object = { id = 1, @@ -63,7 +75,7 @@ g.test_flatten = function() age = 50, } - local tuple, err = utils.flatten(object, space_format, true) + local tuple, err = utils.flatten(object, space_format) t.assert(err == nil) t.assert_equals(tuple, {1, nil, 'Marilyn', 50}) From 854ae0668c0b0deebaba41a8dad735b5076d4a4d Mon Sep 17 00:00:00 2001 From: Sergey Volgin Date: Thu, 24 Sep 2020 17:04:53 +0400 Subject: [PATCH 3/6] Small performance fix --- crud/insert.lua | 7 ++++--- crud/replace.lua | 2 +- crud/update.lua | 5 +++-- crud/upsert.lua | 9 +++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crud/insert.lua b/crud/insert.lua index 673f87de..c79a00bd 100644 --- a/crud/insert.lua +++ b/crud/insert.lua @@ -62,9 +62,10 @@ function insert.call(space_name, obj, opts) if space == nil then return nil, InsertError:new("Space %q doesn't exists", space_name) end + local space_format = space:format() -- compute default buckect_id - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space_format) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end @@ -77,7 +78,7 @@ function insert.call(space_name, obj, opts) return nil, InsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - local tuple, err = utils.flatten(obj, space:format(), bucket_id) + local tuple, err = utils.flatten(obj, space_format, bucket_id) if err ~= nil then return nil, InsertError:new("Object is specified in bad format: %s", err) end @@ -92,7 +93,7 @@ function insert.call(space_name, obj, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, InsertError:new("Received tuple that doesn't match space format: %s", err) end diff --git a/crud/replace.lua b/crud/replace.lua index 80b4a1aa..9f830288 100644 --- a/crud/replace.lua +++ b/crud/replace.lua @@ -90,7 +90,7 @@ function replace.call(space_name, obj, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, ReplaceError:new("Received tuple that doesn't match space format: %s", err) end diff --git a/crud/update.lua b/crud/update.lua index 59c611fd..9dded443 100644 --- a/crud/update.lua +++ b/crud/update.lua @@ -64,12 +64,13 @@ function update.call(space_name, key, user_operations, opts) if space == nil then return nil, UpdateError:new("Space %q doesn't exists", space_name) end + local space_format = space:format() if box.tuple.is(key) then key = key:totable() end - local operations, err = utils.convert_operations(user_operations, space:format()) + local operations, err = utils.convert_operations(user_operations, space_format) if err ~= nil then return nil, UpdateError:new("Wrong operations are specified: %s", err) end @@ -90,7 +91,7 @@ function update.call(space_name, key, user_operations, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, UpdateError:new("Received tuple that doesn't match space format: %s", err) end diff --git a/crud/upsert.lua b/crud/upsert.lua index 46bc59b3..b10ca902 100644 --- a/crud/upsert.lua +++ b/crud/upsert.lua @@ -64,13 +64,14 @@ function upsert.call(space_name, obj, user_operations, opts) return nil, UpsertError:new("Space %q doesn't exists", space_name) end - local operations, err = utils.convert_operations(user_operations, space:format()) + local space_format = space:format() + local operations, err = utils.convert_operations(user_operations, space_format) if err ~= nil then return nil, UpsertError:new("Wrong operations are specified: %s", err) end -- compute default buckect_id - local tuple, err = utils.flatten(obj, space:format()) + local tuple, err = utils.flatten(obj, space_format) if err ~= nil then return nil, UpsertError:new("Object is specified in bad format: %s", err) end @@ -83,7 +84,7 @@ function upsert.call(space_name, obj, user_operations, opts) return nil, UpsertError:new("Failed to get replicaset for bucket_id %s: %s", bucket_id, err.err) end - local tuple, err = utils.flatten(obj, space:format(), bucket_id) + local tuple, err = utils.flatten(obj, space_format, bucket_id) if err ~= nil then return nil, UpsertError:new("Object is specified in bad format: %s", err) end @@ -98,7 +99,7 @@ function upsert.call(space_name, obj, user_operations, opts) end local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space:format()) + local object, err = utils.unflatten(tuple, space_format) if err ~= nil then return nil, UpsertError:new("Received tuple that doesn't match space format: %s", err) end From 1a127274e1233e3101e82a2630fa55e842222995 Mon Sep 17 00:00:00 2001 From: Sergey Volgin Date: Thu, 24 Sep 2020 18:28:56 +0400 Subject: [PATCH 4/6] Change bucket_id name check --- crud/common/utils.lua | 8 ++++---- crud/upsert.lua | 9 ++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 6c89cdac..8c06c8c9 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -48,10 +48,10 @@ function utils.flatten(object, space_format, bucket_id) if not field_format.is_nullable and value == nil then return nil, FlattenError:new("Field %q isn't nullable", field_format.name) end - else - if bucket_id ~= nil then - value = bucket_id - end + end + + if bucket_id ~= nil and field_format.name == 'bucket_id' then + value = bucket_id end tuple[fieldno] = value diff --git a/crud/upsert.lua b/crud/upsert.lua index b10ca902..2a3c8c7f 100644 --- a/crud/upsert.lua +++ b/crud/upsert.lua @@ -98,13 +98,8 @@ function upsert.call(space_name, obj, user_operations, opts) return nil, UpsertError:new("Failed to upsert: %s", err) end - local tuple = results[replicaset.uuid] - local object, err = utils.unflatten(tuple, space_format) - if err ~= nil then - return nil, UpsertError:new("Received tuple that doesn't match space format: %s", err) - end - - return object + --upsert always return nil + return results[replicaset.uuid] end return upsert From 078a8125a0c21a04b177d5f11c30e63f867b2215 Mon Sep 17 00:00:00 2001 From: Sergey Volgin Date: Thu, 24 Sep 2020 19:08:04 +0400 Subject: [PATCH 5/6] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd359762..86d7e1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +* CRUD operations: + * replace + * upsert + ## [0.1.0] - 2020-09-23 ### Added From 93c09d10b2700260438835013c02343ef98abf34 Mon Sep 17 00:00:00 2001 From: savolgin Date: Fri, 25 Sep 2020 14:37:33 +0400 Subject: [PATCH 6/6] remove unused variable --- crud/upsert.lua | 2 +- test/integration/simple_operations_test.lua | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crud/upsert.lua b/crud/upsert.lua index 2a3c8c7f..bddaf663 100644 --- a/crud/upsert.lua +++ b/crud/upsert.lua @@ -19,7 +19,7 @@ local function call_upsert_on_storage(space_name, tuple, operations) local space = box.space[space_name] if space == nil then - return nil, UpdateError:new("Space %q doesn't exists", space_name) + return nil, UpsertError:new("Space %q doesn't exists", space_name) end return space:upsert(tuple, operations) diff --git a/test/integration/simple_operations_test.lua b/test/integration/simple_operations_test.lua index ff5a7805..1506ad06 100644 --- a/test/integration/simple_operations_test.lua +++ b/test/integration/simple_operations_test.lua @@ -3,7 +3,6 @@ local fio = require('fio') local t = require('luatest') local g_memtx = t.group('simple_operations_memtx') local g_vinyl = t.group('simple_operations_vinyl') -local g = t.group('simple_operations') local helpers = require('test.helper')