Skip to content

Commit

Permalink
Fix "Go pointer to Go pointer" panics
Browse files Browse the repository at this point in the history
The Cgo runtime verifies that Go never passes pointers to other Go
pointers, which is required for correct garbage collection.
Unfortunately, its checks are not perfect, and there are occasional
false positives. Our code triggers these false positives if the
slice passed to compression functions is in the same memory
allocation as Go pointers. This happened when trying to use zstd with
another package's Writer type, which has an internal buffer.

The tests added in this PR all fail with the following panic. The
fix is to ensure the expression unsafe.Pointer(&src[0]) is inside the
Cgo call, and not before. This is documented in the following issue:

golang/go#14210 (comment)

Fixes:

... TODO
  • Loading branch information
evanj committed Mar 5, 2021
1 parent 12a1eb7 commit 881fb82
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 0 deletions.
7 changes: 7 additions & 0 deletions zstd_ctx_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@ func TestCtxCompressLevel(t *testing.T) {
}
}

func TestCtxCompressLevelNoGoPointers(t *testing.T) {
testCompressNoGoPointers(t, func(input []byte) ([]byte, error) {
cctx := NewCtx()
return cctx.CompressLevel(nil, input, BestSpeed)
})
}

func TestCtxEmptySliceCompress(t *testing.T) {
ctx := NewCtx()

Expand Down
16 changes: 16 additions & 0 deletions zstd_stream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,22 @@ func TestStreamDecompressionChunks(t *testing.T) {
}
}

func TestStreamWriteNoGoPointers(t *testing.T) {
testCompressNoGoPointers(t, func(input []byte) ([]byte, error) {
buf := &bytes.Buffer{}
zw := NewWriter(buf)
_, err := zw.Write(input)
if err != nil {
return nil, err
}
err = zw.Close()
if err != nil {
return nil, err
}
return buf.Bytes(), nil
})
}

func BenchmarkStreamCompression(b *testing.B) {
if raw == nil {
b.Fatal(ErrNoPayloadEnv)
Expand Down
37 changes: 37 additions & 0 deletions zstd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,43 @@ func TestCompressLevel(t *testing.T) {
}
}

// structWithGoPointers contains a byte buffer and a pointer to Go objects (slice). This means
// Cgo checks can fail when passing a pointer to buffer:
// "panic: runtime error: cgo argument has Go pointer to Go pointer"
// https://github.com/golang/go/issues/14210#issuecomment-346402945
type structWithGoPointers struct {
buffer [1]byte
slice []byte
}

// testCompressDecompressByte ensures that functions use the correct unsafe.Pointer assignment
// to avoid "Go pointer to Go pointer" panics.
func testCompressNoGoPointers(t *testing.T, compressFunc func(input []byte) ([]byte, error)) {
t.Helper()

s := structWithGoPointers{}
s.buffer[0] = 0x42
s.slice = s.buffer[:1]

compressed, err := compressFunc(s.slice)
if err != nil {
t.Fatal(err)
}
decompressed, err := Decompress(nil, compressed)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(decompressed, s.slice) {
t.Errorf("decompressed=%#v input=%#v", decompressed, s.slice)
}
}

func TestCompressLevelNoGoPointers(t *testing.T) {
testCompressNoGoPointers(t, func(input []byte) ([]byte, error) {
return CompressLevel(nil, input, BestSpeed)
})
}

func doCompressLevel(payload []byte, out []byte) error {
out, err := CompressLevel(out, payload, DefaultCompression)
if err != nil {
Expand Down

0 comments on commit 881fb82

Please sign in to comment.