Fix #883: incorrect error handling in lyaml

* Check return codes of libyaml
* Synchronyze check_utf8() logic with libyaml
* Remove analyze_string() - this logic is performed by libyaml
* Add the test cases

test/app/yaml.result

@@ -273,7 +273,7 @@ ok - ucdata
     # compact: end
 ok - compact
     # output
-    1..9
+    1..12
     ok - encode for true
     ok - decode for 'yes'
     ok - encode for false
@@ -282,6 +282,9 @@ ok - compact
     ok - decode for ~
     ok - encode for binary
     ok - encode for binary (2) - gh-354
+    ok - encode for binary (3) - gh-1302
+    ok - encode for binary (4) - gh-883
+    ok - encode for utf-8
     ok - tutorial string
     # output: end
 ok - output

test/app/yaml.test.lua

@@ -42,7 +42,7 @@ local function test_compact(test, s)
 end
 
 local function test_output(test, s)
-    test:plan(9)
+    test:plan(12)
     test:is(s.encode({true}), '---\n- true\n...\n', "encode for true")
     test:is(s.decode("---\nyes\n..."), true, "decode for 'yes'")
     test:is(s.encode({false}), '---\n- false\n...\n', "encode for false")
@@ -53,6 +53,18 @@ local function test_output(test, s)
         "encode for binary")
     test:is(s.encode("\x08\x5c\xc2\x80\x12\x2f"), '--- !!binary CFzCgBIv\n...\n',
         "encode for binary (2) - gh-354")
+    test:is(s.encode("\xe0\x82\x85\x00"), '--- !!binary 4IKFAA==\n...\n',
+        "encode for binary (3) - gh-1302")
+    -- gh-883: console can hang tarantool process
+    local t = {}
+    for i=0x8000,0xffff,1 do
+        table.insert(t, require('pickle').pack( 'i', i ));
+    end
+    local _, count = string.gsub(s.encode(t), "!!binary", "")
+    test:is(count, 30880, "encode for binary (4) - gh-883")
+    test:is(s.encode("фЫр!"), '--- фЫр!\n...\n',
+        "encode for utf-8")
+
     test:is(s.encode("Tutorial -- Header\n====\n\nText"),
         "--- |-\n  Tutorial -- Header\n  ====\n\n  Text\n...\n", "tutorial string");
 end

test/box/net.box.result

@@ -727,28 +727,41 @@ cnc.console ~= nil
 ...
 cnc:console('return 1, 2, 3, "string", nil')
 ---
-- |
-  ---
+- '---
+
   - 1
+
   - 2
+
   - 3
+
   - string
+
   - null
+
   ...
+
+'
 ...
 cnc:console('error("test")')
 ---
-- |
-  ---
+- '---
+
   - error: test
+
   ...
+
+'
 ...
 cnc:console('a = {1, 2, 3, 4}; return a[3]')
 ---
-- |
-  ---
+- '---
+
   - 3
+
   ...
+
+'
 ...
 -- #545 user or password is not defined
 remote:new(LISTEN.host, LISTEN.service, { user = 'test' })