Skip to content

Commit

Permalink
test: enhance data race detection
Browse files Browse the repository at this point in the history
  • Loading branch information
liuq19 committed Jun 14, 2024
1 parent 3b2fa46 commit cf69e48
Show file tree
Hide file tree
Showing 9 changed files with 351 additions and 65 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/unit_test-linux-x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on: push
jobs:
build:
strategy:
max-parallel: 2
matrix:
go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
runs-on: [self-hosted, X64]
Expand All @@ -28,9 +29,15 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Data Race
run: |
./scripts/test_race.sh
- name: PCSP Test
run: python3 ./scripts/test_pcsp.py



- name: Unit Test
run: |
go test -race -covermode=atomic -coverprofile=coverage.txt ./...
Expand Down
73 changes: 73 additions & 0 deletions internal/decoder/decode_norace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
//go:build !race
// +build !race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/utf8`
)

// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)
return err
}
80 changes: 80 additions & 0 deletions internal/decoder/decode_race.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
//go:build race
// +build race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/utf8`
)

func helpDetectDataRace(val interface{}) {
_, _ = json.Marshal(val)
}

// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)

/* put last to make the panic from sonic will always be caught at first */
helpDetectDataRace(val)
return err
}
47 changes: 0 additions & 47 deletions internal/decoder/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,12 @@
package decoder

import (
`unsafe`
`encoding/json`
`reflect`
`runtime`

`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
`github.com/bytedance/sonic/internal/rt`
`github.com/bytedance/sonic/option`
`github.com/bytedance/sonic/utf8`
)

const (
Expand Down Expand Up @@ -106,49 +102,6 @@ func (self *Decoder) CheckTrailings() error {
}


// Decode parses the JSON-encoded data from current position and stores the result
// in the value pointed to by val.
func (self *Decoder) Decode(val interface{}) error {
/* validate json if needed */
if (self.f & (1 << _F_validate_string)) != 0 && !utf8.ValidateString(self.s){
dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
self.s = rt.Mem2Str(dbuf)
}

vv := rt.UnpackEface(val)
vp := vv.Value

/* check for nil type */
if vv.Type == nil {
return &json.InvalidUnmarshalError{}
}

/* must be a non-nil pointer */
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}

etp := rt.PtrElem(vv.Type)

/* check the defined pointer type for issue 379 */
if vv.Type.IsNamed() {
newp := vp
etp = vv.Type
vp = unsafe.Pointer(&newp)
}

/* create a new stack, and call the decoder */
sb := newStack()
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
/* return the stack back */
self.i = nb
freeStack(sb)

/* avoid GC ahead */
runtime.KeepAlive(vv)
return err
}

// UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
// int64 instead of as a float64.
func (self *Decoder) UseInt64() {
Expand Down
44 changes: 44 additions & 0 deletions internal/encoder/encode_norace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
//go:build !race
// +build !race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package encoder

import (
`runtime`

`github.com/bytedance/sonic/internal/rt`
)


func encodeInto(buf *[]byte, val interface{}, opts Options) error {
stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)
return err
}
55 changes: 55 additions & 0 deletions internal/encoder/encode_race.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@

//go:build race
// +build race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package encoder

import (
`encoding/json`
`runtime`

`github.com/bytedance/sonic/internal/rt`
)


func helpDetectDataRace(val interface{}) {
_, _ = json.Marshal(val)
}

func encodeInto(buf *[]byte, val interface{}, opts Options) error {


stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)

/* put last to make the panic from sonic will always be caught at first */
helpDetectDataRace(val)
return err
}
18 changes: 0 additions & 18 deletions internal/encoder/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
`bytes`
`encoding/json`
`reflect`
`runtime`
`unsafe`

`github.com/bytedance/sonic/internal/native`
Expand Down Expand Up @@ -233,23 +232,6 @@ func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
return err
}

func encodeInto(buf *[]byte, val interface{}, opts Options) error {
stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)
return err
}

func encodeFinish(buf []byte, opts Options) []byte {
if opts & EscapeHTML != 0 {
buf = HTMLEscape(nil, buf)
Expand Down
Loading

0 comments on commit cf69e48

Please sign in to comment.