lua: prevent serialization of error for ucdata

Without checking the return value of lua_pcall()` in
`lua_field_inspect_ucdata()`, the error message itself is returned as a
serialized result. The result status of `lua_pcall()` is not ignored
now.

NO_DOC=bugfix

Closes tarantool#9396

(cherry picked from commit 98474f7)

changelogs/unreleased/gh-9396-ucdata-serialize-error.md

@@ -0,0 +1,4 @@
+## bugfix/lua
+
+* An error from a serializer function for cdata and userdata is not ignored now
+  (gh-9396).

src/lua/serializer.c

@@ -274,7 +274,10 @@ lua_field_inspect_ucdata(struct lua_State *L, struct luaL_serializer *cfg,
 		}
 		/* copy object itself */
 		lua_pushvalue(L, idx);
-		lua_pcall(L, 1, 1, 0);
+		if (lua_pcall(L, 1, 1, 0) != LUA_OK) {
+			diag_set(LuajitError, lua_tostring(L, -1));
+			return -1;
+		}
 		/* replace obj with the unpacked value */
 		lua_replace(L, idx);
 		if (luaL_tofield(L, cfg, idx, field) < 0)

test/app-luatest/gh_9396_ucdata_serialize_error_test.lua

@@ -0,0 +1,18 @@
+local t = require('luatest')
+local g = t.group()
+
+-- XXX: Use `yaml.encode()` to trigger a serialization error.
+local yaml = require('yaml')
+
+local ERRMSG = 'error in serializer'
+
+g.test_error_in_ucdata_serializer = function()
+    local ud_mt = {__serialize = function()
+        error(ERRMSG)
+    end}
+    ud_mt.__index = ud_mt
+    local ud = newproxy(true)
+    debug.setmetatable(ud, ud_mt)
+    -- XXX: This helper fails if there is no error raised.
+    t.assert_error_msg_matches('.*' .. ERRMSG, yaml.encode, ud)
+end