Skip to content

Commit

Permalink
test: enhance data race detection in encoder
Browse files Browse the repository at this point in the history
  • Loading branch information
liuq19 committed Jun 18, 2024
1 parent 3b2fa46 commit 5eac274
Show file tree
Hide file tree
Showing 11 changed files with 226 additions and 79 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/unit_test-linux-x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on: push
jobs:
build:
strategy:
max-parallel: 4
matrix:
go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
runs-on: [self-hosted, X64]
Expand All @@ -28,9 +29,15 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Data Race
run: |
./scripts/test_race.sh
- name: PCSP Test
run: python3 ./scripts/test_pcsp.py



- name: Unit Test
run: |
go test -race -covermode=atomic -coverprofile=coverage.txt ./...
Expand Down
31 changes: 0 additions & 31 deletions encoder/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
`runtime`
`runtime/debug`
`strconv`
`sync`
`testing`
`time`

Expand All @@ -47,36 +46,6 @@ func TestMain(m *testing.M) {
m.Run()
}

func TestGC(t *testing.T) {
if debugSyncGC {
return
}
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
n := len(out)
wg := &sync.WaitGroup{}
N := 10000
for i:=0; i<N; i++ {
wg.Add(1)
go func (wg *sync.WaitGroup, size int) {
defer wg.Done()
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Error(err)
return
}
if len(out) != size {
t.Error(len(out), size)
return
}
runtime.GC()
}(wg, n)
}
wg.Wait()
}

type sample struct {
M map[string]interface{}
S []interface{}
Expand Down
44 changes: 44 additions & 0 deletions internal/encoder/encode_norace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
//go:build !race
// +build !race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package encoder

import (
`runtime`

`github.com/bytedance/sonic/internal/rt`
)


func encodeInto(buf *[]byte, val interface{}, opts Options) error {
stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)
return err
}
54 changes: 54 additions & 0 deletions internal/encoder/encode_race.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
//go:build race
// +build race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package encoder

import (
`encoding/json`
`runtime`

`github.com/bytedance/sonic/internal/rt`
)


func helpDetectDataRace(val interface{}) {
_, _ = json.Marshal(val)
}

func encodeInto(buf *[]byte, val interface{}, opts Options) error {


stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)

/* put last to make the panic from sonic will always be caught at first */
helpDetectDataRace(val)
return err
}
18 changes: 0 additions & 18 deletions internal/encoder/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
`bytes`
`encoding/json`
`reflect`
`runtime`
`unsafe`

`github.com/bytedance/sonic/internal/native`
Expand Down Expand Up @@ -233,23 +232,6 @@ func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
return err
}

func encodeInto(buf *[]byte, val interface{}, opts Options) error {
stk := newStack()
efv := rt.UnpackEface(val)
err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))

/* return the stack into pool */
if err != nil {
resetStack(stk)
}
freeStack(stk)

/* avoid GC ahead */
runtime.KeepAlive(buf)
runtime.KeepAlive(efv)
return err
}

func encodeFinish(buf []byte, opts Options) []byte {
if opts & EscapeHTML != 0 {
buf = HTMLEscape(nil, buf)
Expand Down
55 changes: 55 additions & 0 deletions internal/encoder/encoder_norace_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
//go:build !race
// +build !race

/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package encoder

import (
`runtime`

`sync`
`testing`
)

func TestGC(t *testing.T) {
if debugSyncGC {
return
}
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
n := len(out)
wg := &sync.WaitGroup{}
N := 10000
for i:=0; i<N; i++ {
wg.Add(1)
go func (wg *sync.WaitGroup, size int) {
defer wg.Done()
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
if len(out) != size {
t.Fatal(len(out), size)
}
runtime.GC()
}(wg, n)
}
wg.Wait()
}
29 changes: 0 additions & 29 deletions internal/encoder/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
`runtime`
`runtime/debug`
`strconv`
`sync`
`testing`
`time`

Expand All @@ -47,34 +46,6 @@ func TestMain(m *testing.M) {
m.Run()
}

func TestGC(t *testing.T) {
if debugSyncGC {
return
}
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
n := len(out)
wg := &sync.WaitGroup{}
N := 10000
for i:=0; i<N; i++ {
wg.Add(1)
go func (wg *sync.WaitGroup, size int) {
defer wg.Done()
out, err := Encode(_GenericValue, 0)
if err != nil {
t.Fatal(err)
}
if len(out) != size {
t.Fatal(len(out), size)
}
runtime.GC()
}(wg, n)
}
wg.Wait()
}

type sample struct {
M map[string]interface{}
S []interface{}
Expand Down
3 changes: 3 additions & 0 deletions issue_test/issue634_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
//go:build !race
// +build !race

/*
* Copyright 2024 ByteDance Inc.
*
Expand Down
50 changes: 50 additions & 0 deletions issue_test/race_test_go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@

//go:build race
// +build race

/*
* Copyright 2024 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


package issue_test

import (
`github.com/bytedance/sonic`
`testing`
)

type MyFoo struct {
List []*int64
}

func TestRaceEncode(t *testing.T) {
f := &MyFoo{
List: []*int64{new(int64), new(int64)},
}

go func() {
f.List = []*int64{new(int64), new(int64)}
}()

go func() {
sonic.Marshal(f)
}()

// encoding/json always detect data race when enabling `-race` here
// go func() {
// json.Marshal(f)
// }()
}
12 changes: 12 additions & 0 deletions scripts/test_race.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

set -xe
cd ./issue_test
mv race_test_go race_test.go
go test -v -run=TestRaceEncode -race -count=100 . > test_race.log || true
if ! grep -q "WARNING: DATA RACE" ./test_race.log; then
echo "TEST FAILED: should data race here"
exit 1
fi
mv race_test.go race_test_go
rm -vrf test_race.log
2 changes: 1 addition & 1 deletion tools/asm2asm

0 comments on commit 5eac274

Please sign in to comment.