Msgpackffi considers msgpack.cfg options now

Lua serializers have common configuration options
that mostly were not respected in msgpackffi module.
This patch fixes that for several configurations:
encode_use_tostring, encode_invalid_as_nil and
encode_load_metatables.
Options encode/decode_invalid_numbers are suggested
to not be considered in msgpackffi module since lua
processed them properly without this check so adding
it would mean cutting functionality provided by the
language in case this field was set to be false.

Closes #4499

src/lua/msgpack.c

@@ -95,7 +95,6 @@ luamp_decode_extension_default(struct lua_State *L, const char **data)
 		   (unsigned char) **data);
 }
 
-
 void
 luamp_set_decode_extension(luamp_decode_extension_f handler)
 {

src/lua/msgpack.h

@@ -47,7 +47,7 @@ struct mpstream;
 /**
  * Default instance of msgpack serializer (msgpack = require('msgpack')).
  * This instance is used by all box's Lua/C API bindings (e.g. space:replace()).
- * All changes made by msgpack.cfg{} function are also affect box's bindings
+ * All changes made by msgpack.cfg{} function also affect box's bindings
  * (this is a feature).
  */
 extern struct luaL_serializer *luaL_msgpack_default;

src/lua/msgpackffi.lua

@@ -224,9 +224,11 @@ local function encode_r(buf, obj, level)
             return
         end
         local serialize = nil
-        local mt = getmetatable(obj)
-        if mt ~= nil then
-            serialize = mt.__serialize
+        if msgpack.cfg.encode_load_metatables then
+            local mt = getmetatable(obj)
+            if mt ~= nil then
+                serialize = mt.__serialize
+            end
         end
         -- calculate the number of array and map elements in the table
         -- TODO: pairs() aborts JIT
@@ -275,7 +277,19 @@ local function encode_r(buf, obj, level)
             error("can not encode FFI type: '"..ffi.typeof(obj).."'")
         end
     else
-        error("can not encode Lua type: '"..type(obj).."'")
+        if msgpack.cfg.encode_use_tostring then
+            obj = tostring(obj)
+            if obj then
+                goto restart
+            else
+                error("can not encode Lua type: '"..type(obj).."'")
+            end
+        else if msgpack.cfg.encode_invalid_as_nil then
+            encode_nil(buf)
+        else
+            error("can not encode Lua type: '"..type(obj).."'")
+        end
+    end
     end
 end
 

src/lua/utils.h

@@ -185,7 +185,7 @@ struct luaL_serializer {
 	 *  + map - at least one table index is not unsigned integer.
 	 *  + regular array - all array indexes are available.
 	 *  + sparse array - at least one array index is missing.
-	 *  + excessively sparse arrat - the number of values missing
+	 *  + excessively sparse array - the number of values missing
 	 * exceeds the configured ratio.
 	 *
 	 * An array is excessively sparse when **all** the following
@@ -212,7 +212,7 @@ struct luaL_serializer {
 	int encode_sparse_safe;
 	/** Max recursion depth for encoding (MsgPack, CJSON only) */
 	int encode_max_depth;
-	/** Enables encoding of NaN and Inf numbers */
+	/** Enables encoding of NaN and Inf numbers (YAML, CJSON only) */
 	int encode_invalid_numbers;
 	/** Floating point numbers precision (YAML, CJSON only) */
 	int encode_number_precision;
@@ -233,7 +233,7 @@ struct luaL_serializer {
 	/** Use NULL for all unrecognizable types */
 	int encode_invalid_as_nil;
 
-	/** Enables decoding NaN and Inf numbers */
+	/** Enables decoding NaN and Inf numbers (YAML, CJSON only) */
 	int decode_invalid_numbers;
 	/** Save __serialize meta-value for decoded arrays and maps */
 	int decode_save_metatables;