Skip to content

Commit

Permalink
feat (encoder): add encoder option NoNullSliceOrMap (#218)
Browse files Browse the repository at this point in the history 
* feat (encoder): add encoder option `NoNullSliceOrMap`

* feat: add option on `sonic.Config`

* build: specify `self-host` to x64 machine
  • Loading branch information
@AsterDY
AsterDY committed Jul 8, 2022
1 parent 08c7640 commit 07d7b86
Show file tree
Hide file tree
Showing 9 changed files with 161 additions and 25 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/license-check.yml
View file
Expand Up @@ -4,11 +4,11 @@ on: push

jobs:
build:
runs-on: self-hosted
runs-on: [self-hosted, X64]
steps:
- uses: actions/checkout@v2

- name: Check License Header
uses: apache/skywalking-eyes@main
uses: apache/skywalking-eyes/header@main
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1 change: 1 addition & 0 deletions api.go
View file
Expand Up @@ -31,6 +31,7 @@ import (
UseUnicodeErrors bool
DisallowUnknownFields bool
CopyString bool
NoNullSliceOrMap bool
}

var (
Expand Down
18 changes: 17 additions & 1 deletion encode_test.go
View file
Expand Up @@ -19,13 +19,13 @@
package sonic

import (
`os`
`bytes`
`encoding`
`encoding/json`
`fmt`
`log`
`math`
`os`
`reflect`
`regexp`
`runtime`
Expand All @@ -36,6 +36,7 @@ import (
`unsafe`

`github.com/bytedance/sonic/encoder`
`github.com/stretchr/testify/assert`
)

var (
Expand Down Expand Up @@ -1153,3 +1154,18 @@ func TestMarshalerError(t *testing.T) {
}
}
}

func TestMarshalNullNil(t *testing.T) {
var v = struct {
A []int
B map[string]int
}{}
o, e := Marshal(v)
assert.Nil(t, e)
assert.Equal(t, `{"A":null,"B":null}`, string(o))
o, e = Config{
NoNullSliceOrMap: true,
}.Froze().Marshal(v)
assert.Nil(t, e)
assert.Equal(t, `{"A":[],"B":{}}`, string(o))
}
38 changes: 33 additions & 5 deletions encoder/assembler_amd64_go116.go
View file
Expand Up @@ -89,11 +89,13 @@ const (
)

const (
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_mulv = -0x5555555555555555
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_array = 0x5d5b // '[]'
_IM_object = 0x7d7b // '{}'
_IM_mulv = -0x5555555555555555
)

const (
Expand Down Expand Up @@ -204,6 +206,8 @@ func (self *_Assembler) compile() {

var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_null : (*_Assembler)._asm_OP_null,
_OP_empty_arr : (*_Assembler)._asm_OP_empty_arr,
_OP_empty_obj : (*_Assembler)._asm_OP_empty_obj,
_OP_bool : (*_Assembler)._asm_OP_bool,
_OP_i8 : (*_Assembler)._asm_OP_i8,
_OP_i16 : (*_Assembler)._asm_OP_i16,
Expand Down Expand Up @@ -767,6 +771,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
}

func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_arr_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_arr_end_{n}")
self.Link("_empty_arr_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_arr_end_{n}")
}

func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_obj_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_obj_end_{n}")
self.Link("_empty_obj_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_obj_end_{n}")
}

func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
Expand Down
50 changes: 39 additions & 11 deletions encoder/assembler_amd64_go117.go
View file
Expand Up @@ -91,11 +91,13 @@ const (
)

const (
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_mulv = -0x5555555555555555
_IM_null = 0x6c6c756e // 'null'
_IM_true = 0x65757274 // 'true'
_IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
_IM_open = 0x00225c22 // '"\"∅'
_IM_array = 0x5d5b // '[]'
_IM_object = 0x7d7b // '{}'
_IM_mulv = -0x5555555555555555
)

const (
Expand Down Expand Up @@ -130,16 +132,16 @@ var (
)

var (
_ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go...
_RP = jit.Reg("DI")
_RL = jit.Reg("SI")
_RC = jit.Reg("DX")
_ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go...
_RP = jit.Reg("DI")
_RL = jit.Reg("SI")
_RC = jit.Reg("DX")
)

var (
_LR = jit.Reg("R9")
_ET = jit.Reg("AX")
_EP = jit.Reg("BX")
_ET = jit.Reg("AX")
_EP = jit.Reg("BX")
)

var (
Expand Down Expand Up @@ -209,6 +211,8 @@ func (self *_Assembler) compile() {

var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
_OP_null : (*_Assembler)._asm_OP_null,
_OP_empty_arr : (*_Assembler)._asm_OP_empty_arr,
_OP_empty_obj : (*_Assembler)._asm_OP_empty_obj,
_OP_bool : (*_Assembler)._asm_OP_bool,
_OP_i8 : (*_Assembler)._asm_OP_i8,
_OP_i16 : (*_Assembler)._asm_OP_i16,
Expand Down Expand Up @@ -780,6 +784,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
}

func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_arr_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_arr_end_{n}")
self.Link("_empty_arr_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_arr_end_{n}")
}

func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
self.Sjmp("JC", "_empty_obj_{n}")
self._asm_OP_null(nil)
self.Sjmp("JMP", "_empty_obj_end_{n}")
self.Link("_empty_obj_{n}")
self.check_size(2)
self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))
self.Emit("ADDQ", jit.Imm(2), _RL)
self.Link("_empty_obj_end_{n}")
}

func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
self.Sjmp("JE" , "_false_{n}") // JE _false_{n}
Expand Down
14 changes: 9 additions & 5 deletions encoder/compiler.go
View file
Expand Up @@ -32,6 +32,8 @@ type _Op uint8

const (
_OP_null _Op = iota + 1
_OP_empty_arr
_OP_empty_obj
_OP_bool
_OP_i8
_OP_i16
Expand Down Expand Up @@ -93,6 +95,8 @@ const (

var _OpNames = [256]string {
_OP_null : "null",
_OP_empty_arr : "empty_arr",
_OP_empty_obj : "empty_obj",
_OP_bool : "bool",
_OP_i8 : "i8",
_OP_i16 : "i16",
Expand Down Expand Up @@ -481,19 +485,19 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
}
}

func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, fn func(*_Program, int, reflect.Type)) {
func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) {
x := p.pc()
p.add(_OP_is_nil)
fn(p, sp, vt)
e := p.pc()
p.add(_OP_goto)
p.pin(x)
p.add(_OP_null)
p.add(nil_op)
p.pin(e)
}

func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compilePtrBody)
self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody)
}

func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
Expand All @@ -505,7 +509,7 @@ func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
}

func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compileMapBody)
self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody)
}

func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) {
Expand Down Expand Up @@ -591,7 +595,7 @@ func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) {
}

func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
self.compileNil(p, sp, vt, self.compileSliceBody)
self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody)
}

func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) {
Expand Down
8 changes: 7 additions & 1 deletion encoder/encoder.go
View file
Expand Up @@ -37,6 +37,7 @@ const (
bitEscapeHTML
bitCompactMarshaler
bitNoQuoteTextMarshaler
bitNoNullSliceOrMap
)

const (
Expand All @@ -52,12 +53,17 @@ const (

// CompactMarshaler indicates that the output JSON from json.Marshaler
// is always compact and needs no validation
CompactMarshaler Options = 1 << bitCompactMarshaler
CompactMarshaler Options = 1 << bitCompactMarshaler

// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler
// is always escaped string and needs no quoting
NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler

// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
// instead of 'null'
NoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap

// CompatibleWithStd is used to be compatible with std encoder.
CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
)

Expand Down
50 changes: 50 additions & 0 deletions encoder/encoder_test.go
View file
Expand Up @@ -76,6 +76,56 @@ func TestGC(t *testing.T) {
wg.Wait()
}

type sample struct {
M map[string]interface{}
S []interface{}
A [0]interface{}
MP *map[string]interface{}
SP *[]interface{}
AP *[0]interface{}
}

func TestOptionSliceOrMapNoNull(t *testing.T) {
obj := sample{}
out, err := Encode(obj, NoNullSliceOrMap)
if err != nil {
t.Fatal(err)
}
require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))

obj2 := sample{}
out, err = Encode(obj2, 0)
if err != nil {
t.Fatal(err)
}
require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
}

func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
b.Run("true", func (b *testing.B) {
obj := sample{}
_, err := Encode(obj, NoNullSliceOrMap)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i:=0;i<b.N;i++{
_, _ = Encode(obj, NoNullSliceOrMap)
}
})

b.Run("false", func (b *testing.B) {
obj2 := sample{}
_, err := Encode(obj2, 0)
if err != nil {
b.Fatal(err)
}
for i:=0;i<b.N;i++{
_, _ = Encode(obj2, 0)
}
})
}

func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
require.Equal(t, exp, fn(arg))
}
Expand Down
3 changes: 3 additions & 0 deletions sonic.go
View file
Expand Up @@ -85,6 +85,9 @@ func (cfg Config) Froze() API {
if cfg.CopyString {
api.decoderOpts |= decoder.OptionCopyString
}
if cfg.NoNullSliceOrMap {
api.encoderOpts |= encoder.NoNullSliceOrMap
}
return api
}

Expand Down

0 comments on commit 07d7b86

Please sign in to comment.