Unexpected varbinary encoding

[DifferentialOrange]

I've met some unexpecting varbinary encoding results while comparing different Tarantool connectors. Since I'm not experienced in both Go and Tarantool-Go usage, there is a high possibility that something wrong with my test runs.

Let Tarantool be

box.cfg{ listen = 3401 }
box.schema.user.grant('guest', 'read,write,execute,create,drop,alter', 'universe')

-- varbinary test

box.schema.space.create('testvarbin')
box.space.testvarbin:create_index('pk', { type = 'tree', parts = {{ field = 1, type = 'varbinary' }} })

buffer = require('buffer')
ffi = require('ffi')

function encode_bin(bytes)
    local tmpbuf = buffer.ibuf()
    local p = tmpbuf:alloc(3 + #bytes)
    p[0] = 0x91
    p[1] = 0xC4
    p[2] = #bytes
    for i, c in pairs(bytes) do
        p[i + 3 - 1] = c
    end
    return tmpbuf
end

function bintuple_insert(space, bytes)
    local tmpbuf = encode_bin(bytes)
    ffi.cdef[[int box_insert(uint32_t space_id, const char *tuple, const char *tuple_end, box_tuple_t **result);]]
    ffi.C.box_insert(space.id, tmpbuf.rpos, tmpbuf.wpos, nil)
end

bintuple_insert(box.space.testvarbin, {0xDE, 0xAD, 0xBE, 0xAF})

tarantool/go-tarantool

package main

import (
	"log"
	"encoding/binary"
	"github.com/tarantool/go-tarantool"
)

func TestVarbinaryTarantool(uri string, user string) {
	opts := tarantool.Opts{User: user}
	conn, err := tarantool.Connect(uri, opts)

	if err != nil {
		log.Fatalf("Connection refused:", err)
	}

	resp, err := conn.Ping()
	log.Println(resp.Code)
	log.Println(resp.Data)
	log.Println(err)

	resp1, err1 := conn.Eval("return box.space.testvarbin:select{}", []interface{}{})

	log.Println("Error", err1)
	log.Println("Code", resp1.Code)
	log.Println("Data", resp1.Data)

	buf := make([]byte, 4)
	binary.BigEndian.PutUint16(buf[0:], 0xa22c)
	binary.BigEndian.PutUint16(buf[2:], 0x04af)

	resp2, err2 := conn.Insert("testvarbin", []interface{}{buf})

	log.Println("Error", err2)
	log.Println("Code", resp2.Code)
	log.Println("Data", resp2.Data)
}

package main

func main() {
    var uri string = "127.0.0.1:3401"
    var user string = "guest"

    TestVarbinaryTarantool(uri, user)
}

Output is

2021/12/20 18:00:13 0
2021/12/20 18:00:13 []
2021/12/20 18:00:13 <nil>
2021/12/20 18:00:13 Error <nil>
2021/12/20 18:00:13 Code 0
2021/12/20 18:00:13 Data [[[[222 173 190 175]]]]
2021/12/20 18:00:13 Error <nil>
2021/12/20 18:00:13 Code 0
2021/12/20 18:00:13 Data [[[162 44 4 175]]]

viciious/go-tarantool

package main

import (
	"context"
	"encoding/binary"
	"log"
	"github.com/viciious/go-tarantool"
)

func TestVarbinaryViciious(uri string, user string) {
	opts := tarantool.Options{User: user}
	conn, err := tarantool.Connect(uri, &opts)

	if err != nil {
		log.Fatalf("Connection refused:", err.Error())
	}

	resp := conn.Exec(context.Background(), &tarantool.Ping{})
	log.Println("Error", resp.Error)
	log.Println("Data", resp.Data)

	expr := "return box.space.testvarbin:select{}"
	q := &tarantool.Eval{Expression: expr}
	resp1 := conn.Exec(context.Background(), q)

	log.Println("Error", resp1.Error)
	log.Println("Data", resp1.Data)

	buf := make([]byte, 4)
	binary.BigEndian.PutUint16(buf[0:], 0xa23c)
	binary.BigEndian.PutUint16(buf[2:], 0x04af)

	query := &tarantool.Insert{Space: "testvarbin", Tuple: []interface{}{buf}}
	resp2 := conn.Exec(context.Background(), query)

	log.Println("Error", resp2.Error)
	log.Println("Data", resp2.Data)
}

package main

func main() {
    var uri string = "127.0.0.1:3401"
    var user string = "guest"

    TestVarbinaryViciious(uri, user)
}

Output is

2021/12/20 18:04:40 Error <nil>
2021/12/20 18:04:40 Data []
2021/12/20 18:04:40 Error <nil>
2021/12/20 18:04:40 Data [[[[222 173 190 175]]]]
2021/12/20 18:04:40 Error <nil>
2021/12/20 18:04:40 Data [[[162 60 4 175]]]

FZambia/tarantool

package main

import (
	"encoding/binary"
	"log"
	"github.com/FZambia/tarantool"
)

func TestVarbinaryFZambia(uri string, user string) {
	opts := tarantool.Opts{User: user}
	conn, errc := tarantool.Connect(uri, opts)
	if errc != nil {
		log.Fatalf("Connection refused: %v", errc)
	}

	resp, err := conn.Exec(tarantool.Ping())

	log.Println("Ping Code", resp.Code)
	log.Println("Ping Data", resp.Data)
	log.Println("Ping Error", err)

	resp1, err1 := conn.Exec(tarantool.Eval("return box.space.testvarbin:select{}", []interface{}{}))

	log.Println("Error", err1)
	log.Println("Code", resp1.Code)
	log.Println("Data", resp1.Data)

	buf := make([]byte, 4)
	binary.BigEndian.PutUint16(buf[0:], 0xa25c)
	binary.BigEndian.PutUint16(buf[2:], 0x04af)

	resp2, err2 := conn.Exec(tarantool.Insert("testvarbin", []interface{}{buf}))

	log.Println("Error", err2)
	log.Println("Code", resp2.Code)
	log.Println("Data", resp2.Data)
}

package main

func main() {
    var uri string = "127.0.0.1:3401"
    var user string = "guest"

    TestVarbinaryFZambia(uri, user)
}

Output is

2021/12/20 18:07:30 Ping Code 0
2021/12/20 18:07:30 Ping Data []
2021/12/20 18:07:30 Ping Error <nil>
2021/12/20 18:07:30 Error <nil>
2021/12/20 18:07:30 Code 0
2021/12/20 18:07:30 Data [[[ޭ��]]]
2021/12/20 18:07:30 Error <nil>
2021/12/20 18:07:30 Code 0
2021/12/20 18:07:30 Data [[�\�]]

I'm not sure how to interpret this, but first two seems like they are ok with varbinary and the last one may have some issues.

[FZambia]

Hello @DifferentialOrange

Uff - that was tricky! The reason is that msgpack v5 decodes types to smallest representation by default. It's not so handy so to mimic msgpack v2 behavior from official go-tarantool library we are using decoder.UseLooseInterfaceDecoding(true) here.

The rules for it from Msgpack v5 docs:

//   - int8, int16, and int32 are converted to int64,
//   - uint8, uint16, and uint32 are converted to uint64,
//   - float32 is converted to float64.
//   - []byte is converted to string.

So []bytereturned from Tarantool is converted to string. But the string is not a valid unicode byte sequence - so it's displayed this way. But this string contains expected bytes.

So two approaches are possible to deal with this:

type varbinary []byte

func (b *varbinary) DecodeMsgpack(d *msgpack.Decoder) error {
	var err error
	*b, err = d.DecodeBytes()
	return err
}

type T struct {
	Value varbinary
}

func TestVarbinary(t *testing.T) {
	conn, err := Connect("127.0.0.1:3301", Opts{
		User: "guest",
	})
	if err != nil {
		t.Fatal(err)
	}

	// Way 1 - with decoding into interface.
	resp, err := conn.Exec(Eval("return box.space.testvarbin:select{}", []interface{}{}))
	if err != nil {
		t.Fatal(err)
	}
	log.Println("Code", resp.Code)
        // Slice bounds check skipped here to keep example short!
	log.Println("Data", []byte(resp.Data[0].([]interface{})[0].([]interface{})[0].(string)))

	// Way 2 - with decoding directly to varbinary.
	var result [][]T
	err = conn.ExecTyped(Eval("return box.space.testvarbin:select{}", []interface{}{}), &result)
	log.Println("Error", err)
        // Slice bounds check skipped here to keep example short!
	log.Println("Data", result[0][0].Value)
}

Output:

❯ go test -run TestVarbinary
2021/12/24 22:23:58 Error <nil>
2021/12/24 22:23:58 Code 0
2021/12/24 22:23:58 Data [222 173 190 175]
2021/12/24 22:23:58 Error <nil>
2021/12/24 22:23:58 Data [222 173 190 175]
PASS
ok  	github.com/FZambia/tarantool	0.104s

[FZambia]

I suppose can be closed.