Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: TestRWMutex is flaky? #20903

Closed
hirochachacha opened this issue Jul 4, 2017 · 12 comments

Comments

Projects
None yet
5 participants
@hirochachacha
Copy link
Contributor

commented Jul 4, 2017

Please answer these questions before submitting your issue. Thanks!

What did you do?

If possible, provide a recipe for reproducing the error.
A complete runnable program is good.
A link on play.golang.org is best.

for i in `seq 100`; do go test -v runtime -run "TestRWMutex"; done

What did you expect to see?

PASS * 100

What did you see instead?

=== RUN   TestRWMutex
SIGQUIT: quit
PC=0x10625b6 m=2 sigcode=0

goroutine 0 [idle]:
runtime.usleep(0x8b00002710, 0x0, 0x68c71d5b2f, 0x45d964b800, 0x271000000000, 0x8bb25b57c4, 0x45d964b800, 0x3, 0x0, 0x68c71d5b2f, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:323 +0x36 fp=0x700006055de0 sp=0x700006055dc0 pc=0x10625b6
runtime.sysmon()
	/Users/hiro/go/src/runtime/proc.go:3799 +0xa8 fp=0x700006055e68 sp=0x700006055de0 pc=0x1039a08
runtime.mstart1()
	/Users/hiro/go/src/runtime/proc.go:1172 +0x11e fp=0x700006055e90 sp=0x700006055e68 pc=0x103285e
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1142 +0x64 fp=0x700006055ea8 sp=0x700006055e90 pc=0x1032734

goroutine 1 [chan receive]:
runtime.gopark(0x131a290, 0xc420076418, 0x1308f6e, 0xc, 0xc420051b17, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc420051ac8 sp=0xc420051a98 pc=0x102fa7c
runtime.goparkunlock(0xc420076418, 0x1308f6e, 0xc, 0xc4200e8017, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc420051b08 sp=0xc420051ac8 pc=0x102fb6e
runtime.chanrecv(0xc4200763c0, 0x0, 0xc420051b01, 0x10ebc14)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc420051bb8 sp=0xc420051b08 pc=0x1005dc4
runtime.chanrecv1(0xc4200763c0, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc420051be8 sp=0xc420051bb8 pc=0x1005a6b
testing.(*T).Run(0xc4200e2000, 0x130879c, 0xb, 0x131b2e0, 0x108f201)
	/Users/hiro/go/src/testing/testing.go:801 +0x332 fp=0xc420051c98 sp=0xc420051be8 pc=0x10ebc32
testing.runTests.func1(0xc4200e2000)
	/Users/hiro/go/src/testing/testing.go:1053 +0x64 fp=0xc420051ce8 sp=0xc420051c98 pc=0x10efd04
testing.tRunner(0xc4200e2000, 0xc420051d98)
	/Users/hiro/go/src/testing/testing.go:754 +0xd0 fp=0xc420051d10 sp=0xc420051ce8 pc=0x10eb8a0
testing.runTests(0xc42000c300, 0x1498000, 0xd7, 0xd7, 0x5f)
	/Users/hiro/go/src/testing/testing.go:1051 +0x2d8 fp=0xc420051dc8 sp=0xc420051d10 pc=0x10ed688
testing.(*M).Run(0xc420051f18, 0x12474b6)
	/Users/hiro/go/src/testing/testing.go:932 +0x111 fp=0xc420051ec0 sp=0xc420051dc8 pc=0x10ec031
runtime_test.TestMain(0xc420051f18)
	/Users/hiro/go/src/runtime/crash_test.go:28 +0x2f fp=0xc420051f10 sp=0xc420051ec0 pc=0x1216c9f
main.main()
	runtime/_test/_testmain.go:902 +0xdb fp=0xc420051f80 sp=0xc420051f10 pc=0x126701b
runtime.main()
	/Users/hiro/go/src/runtime/proc.go:185 +0x20d fp=0xc420051fe0 sp=0xc420051f80 pc=0x102f5dd
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420051fe8 sp=0xc420051fe0 pc=0x10612f1

goroutine 2 [force gc (idle)]:
runtime.gopark(0x131a290, 0x149a0c0, 0x130a49c, 0xf, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002c768 sp=0xc42002c738 pc=0x102fa7c
runtime.goparkunlock(0x149a0c0, 0x130a49c, 0xf, 0xc420000114, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002c7a8 sp=0xc42002c768 pc=0x102fb6e
runtime.forcegchelper()
	/Users/hiro/go/src/runtime/proc.go:235 +0xcc fp=0xc42002c7e0 sp=0xc42002c7a8 pc=0x102f89c
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002c7e8 sp=0xc42002c7e0 pc=0x10612f1
created by runtime.init.4
	/Users/hiro/go/src/runtime/proc.go:224 +0x35

goroutine 3 [GC sweep wait]:
runtime.gopark(0x131a290, 0x149a3c0, 0x130941f, 0xd, 0x1021014, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002cf60 sp=0xc42002cf30 pc=0x102fa7c
runtime.goparkunlock(0x149a3c0, 0x130941f, 0xd, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002cfa0 sp=0xc42002cf60 pc=0x102fb6e
runtime.bgsweep(0xc420020070)
	/Users/hiro/go/src/runtime/mgcsweep.go:52 +0xa3 fp=0xc42002cfd8 sp=0xc42002cfa0 pc=0x10210d3
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002cfe0 sp=0xc42002cfd8 pc=0x10612f1
created by runtime.gcenable
	/Users/hiro/go/src/runtime/mgc.go:216 +0x58

goroutine 4 [finalizer wait]:
runtime.gopark(0x131a290, 0x14b9f10, 0x1309da5, 0xe, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002d700 sp=0xc42002d6d0 pc=0x102fa7c
runtime.goparkunlock(0x14b9f10, 0x1309da5, 0xe, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002d740 sp=0xc42002d700 pc=0x102fb6e
runtime.runfinq()
	/Users/hiro/go/src/runtime/mfinal.go:175 +0xb8 fp=0xc42002d7e0 sp=0xc42002d740 pc=0x1017ed8
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002d7e8 sp=0xc42002d7e0 pc=0x10612f1
created by runtime.createfing
	/Users/hiro/go/src/runtime/mfinal.go:156 +0x62

goroutine 5 [syscall]:
runtime.notetsleepg(0x14ba2c0, 0xffffffffffffffff, 0x0)
	/Users/hiro/go/src/runtime/lock_sema.go:280 +0x4b fp=0xc42002df80 sp=0xc42002df40 pc=0x10119eb
os/signal.signal_recv(0x0)
	/Users/hiro/go/src/runtime/sigqueue.go:131 +0xa7 fp=0xc42002dfa8 sp=0xc42002df80 pc=0x1045807
os/signal.loop()
	/Users/hiro/go/src/os/signal/signal_unix.go:22 +0x22 fp=0xc42002dfe0 sp=0xc42002dfa8 pc=0x10e4232
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002dfe8 sp=0xc42002dfe0 pc=0x10612f1
created by os/signal.init.0
	/Users/hiro/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 6 [select, locked to thread]:
runtime.gopark(0x131a2d8, 0x0, 0x1306abf, 0x6, 0x18, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42003aca0 sp=0xc42003ac70 pc=0x102fa7c
runtime.selectgo(0xc42003af50, 0xc420076180)
	/Users/hiro/go/src/runtime/select.go:395 +0x1138 fp=0xc42003af18 sp=0xc42003aca0 pc=0x103fff8
runtime.ensureSigM.func1()
	/Users/hiro/go/src/runtime/signal_unix.go:511 +0x1fe fp=0xc42003afe0 sp=0xc42003af18 pc=0x105da9e
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42003afe8 sp=0xc42003afe0 pc=0x10612f1
created by runtime.ensureSigM
	/Users/hiro/go/src/runtime/signal_unix.go:494 +0xda

goroutine 7 [chan receive]:
runtime.gopark(0x131a290, 0xc42005e358, 0x1308f6e, 0xc, 0xc420024117, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002ee80 sp=0xc42002ee50 pc=0x102fa7c
runtime.goparkunlock(0xc42005e358, 0x1308f6e, 0xc, 0x17, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002eec0 sp=0xc42002ee80 pc=0x102fb6e
runtime.chanrecv(0xc42005e300, 0x0, 0x1, 0x0)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc42002ef70 sp=0xc42002eec0 pc=0x1005dc4
runtime.chanrecv1(0xc42005e300, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc42002efa0 sp=0xc42002ef70 pc=0x1005a6b
testing.(*M).before.func1(0xc42005e300)
	/Users/hiro/go/src/testing/testing.go:1111 +0x38 fp=0xc42002efd8 sp=0xc42002efa0 pc=0x10efd98
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002efe0 sp=0xc42002efd8 pc=0x10612f1
created by testing.(*M).before
	/Users/hiro/go/src/testing/testing.go:1110 +0x191

goroutine 8 [chan receive]:
runtime.gopark(0x131a290, 0xc420106058, 0x1308f6e, 0xc, 0xc42002f617, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002f5e8 sp=0xc42002f5b8 pc=0x102fa7c
runtime.goparkunlock(0xc420106058, 0x1308f6e, 0xc, 0x1010f0120104017, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002f628 sp=0xc42002f5e8 pc=0x102fb6e
runtime.chanrecv(0xc420106000, 0x0, 0xc42002f701, 0x1249879)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc42002f6d8 sp=0xc42002f628 pc=0x1005dc4
runtime.chanrecv1(0xc420106000, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc42002f708 sp=0xc42002f6d8 pc=0x1005a6b
runtime_test.HammerRWMutex(0x4, 0xa, 0x3e8)
	/Users/hiro/go/src/runtime/rwmutex_test.go:103 +0x1eb fp=0xc42002f770 sp=0xc42002f708 pc=0x12498ab
runtime_test.TestRWMutex(0xc4200e20f0)
	/Users/hiro/go/src/runtime/rwmutex_test.go:118 +0x12f fp=0xc42002f7a8 sp=0xc42002f770 pc=0x1249a0f
testing.tRunner(0xc4200e20f0, 0x131b2e0)
	/Users/hiro/go/src/testing/testing.go:754 +0xd0 fp=0xc42002f7d0 sp=0xc42002f7a8 pc=0x10eb8a0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002f7d8 sp=0xc42002f7d0 pc=0x10612f1
created by testing.(*T).Run
	/Users/hiro/go/src/testing/testing.go:800 +0x314

goroutine 50 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:92 +0xc1

goroutine 51 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 52 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010efc0 sp=0xc42010efb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010efc8 sp=0xc42010efc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 53 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010f7c0 sp=0xc42010f7b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010f7c8 sp=0xc42010f7c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 54 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 55 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 56 [runnable]:
runtime_test.writer(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:71 fp=0xc420110fc0 sp=0xc420110fb8 pc=0x1249540
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420110fc8 sp=0xc420110fc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:97 +0x16d

goroutine 57 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc4201117c0 sp=0xc4201117b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4201117c8 sp=0xc4201117c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 58 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc420111fc0 sp=0xc420111fb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420111fc8 sp=0xc420111fc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 59 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc4200f47c0 sp=0xc4200f47b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4200f47c8 sp=0xc4200f47c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 60 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010afc0 sp=0xc42010afb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010afc8 sp=0xc42010afc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 61 [runnable]:
runtime.(*RWMutex).RUnlock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:361 +0x3c fp=0xc42010b748 sp=0xc42010b740 pc=0x105a9bc
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:66 +0x45 fp=0xc42010b7c0 sp=0xc42010b748 pc=0x1249405
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010b7c8 sp=0xc42010b7c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

rax    0x4
rbx    0x3
rcx    0x700006055dc0
rdx    0x0
rdi    0x0
rsi    0x0
rbp    0x700006055dd0
rsp    0x700006055dc0
r8     0x700006055dc0
r9     0x8
r10    0x0
r11    0x246
r12    0x3b4a277fed96
r13    0xb03
r14    0x10326d0
r15    0x1602620
rip    0x10625b6
rflags 0x247
cs     0x7
fs     0x0
gs     0x0
*** Test killed with quit: ran too long (10m0s).
FAIL	runtime	600.024s

Does this issue reproduce with the latest release (go1.8.3)?

System details

go version devel +a89e6be5e4 Mon Jul 3 14:08:01 2017 +0000 darwin/amd64
GOARCH="amd64"
GOBIN=""
GOEXE=""
GOHOSTARCH="amd64"
GOHOSTOS="darwin"
GOOS="darwin"
GOPATH="/Users/hiro/.go"
GORACE=""
GOROOT="/Users/hiro/go"
GOTOOLDIR="/Users/hiro/go/pkg/tool/darwin_amd64"
GCCGO="gccgo"
CC="clang"
GOGCCFLAGS="-fPIC -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/var/folders/wq/dwn8hs0x7njbzty9f68y61700000gn/T/go-build091579411=/tmp/go-build -gno-record-gcc-switches -fno-common"
CXX="clang++"
CGO_ENABLED="1"
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOROOT/bin/go version: go version devel +a89e6be5e4 Mon Jul 3 14:08:01 2017 +0000 darwin/amd64
GOROOT/bin/go tool compile -V: compile version devel +a89e6be5e4 Mon Jul 3 14:08:01 2017 +0000 X:framepointer
uname -v: Darwin Kernel Version 16.6.0: Fri Apr 14 16:21:16 PDT 2017; root:xnu-3789.60.24~6/RELEASE_X86_64
ProductName:	Mac OS X
ProductVersion:	10.12.5
BuildVersion:	16F73
lldb --version: lldb-370.0.42
  Swift-3.1
gdb --version: GNU gdb (GDB) 7.12.1

FWIW, sync.TestRWMutex doesn't fail.

@davecheney

This comment has been minimized.

Copy link
Contributor

commented Jul 4, 2017

@hirochachacha

This comment has been minimized.

Copy link
Contributor Author

commented Jul 4, 2017

runtime.TestRWMutex was introduced few weeks ago.

@hirochachacha

This comment has been minimized.

Copy link
Contributor Author

commented Jul 5, 2017

can you please try bisecting.

8083297 is a suspect.

@davecheney

This comment has been minimized.

Copy link
Contributor

commented Jul 5, 2017

@bradfitz bradfitz added this to the Go1.9Maybe milestone Jul 5, 2017

@bradfitz

This comment has been minimized.

Copy link
Member

commented Jul 5, 2017

@aclements, can you look at this?

@aclements

This comment has been minimized.

Copy link
Member

commented Jul 5, 2017

@hirochachacha, thanks for the report. I haven't been able to reproduce this on either linux/amd64 or darwin/amd64 yet.

How many CPUs does your system have? From the traceback it looks like at least 5 (so I assume really 6 or 8).

Can you reproduce it with GOTRACEBACK=crash set and post the output?

@hirochachacha

This comment has been minimized.

Copy link
Contributor Author

commented Jul 6, 2017

How many CPUs does your system have? From the traceback it looks like at least 5 (so I assume really 6 or 8).

It's only 4.

$ sysctl hw | grep cpu
hw.ncpu: 4
hw.activecpu: 4
hw.cpufrequency_max: 1800000000
hw.cpufrequency_min: 1800000000
hw.cpufrequency: 1800000000
hw.cpufamily: 526772277
hw.cpu64bit_capable: 1
hw.cpusubtype: 4
hw.cputype: 7
hw.logicalcpu_max: 4
hw.logicalcpu: 4
hw.physicalcpu_max: 2
hw.physicalcpu: 2
hw.cputhreadtype: 1

Can you reproduce it with GOTRACEBACK=crash set and post the output?

$ GOTRACEBACK=crash go test -v runtime -run "TestRWMutex"
=== RUN   TestRWMutex
SIGQUIT: quit
PC=0x10625b6 m=2 sigcode=0

goroutine 0 [idle]:
runtime.usleep(0x8b00002710, 0x0, 0x68c779c709, 0x45d964b800, 0x271000000000, 0x8bb2880fa8, 0x45d964b800, 0x3, 0x0, 0x68c779c709, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:323 +0x36 fp=0x700005e8dde0 sp=0x700005e8ddc0 pc=0x10625b6
runtime.sysmon()
	/Users/hiro/go/src/runtime/proc.go:3799 +0xa8 fp=0x700005e8de68 sp=0x700005e8dde0 pc=0x1039a08
runtime.mstart1()
	/Users/hiro/go/src/runtime/proc.go:1172 +0x11e fp=0x700005e8de90 sp=0x700005e8de68 pc=0x103285e
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1142 +0x64 fp=0x700005e8dea8 sp=0x700005e8de90 pc=0x1032734

goroutine 1 [chan receive]:
runtime.gopark(0x131a290, 0xc420074418, 0x1308f6e, 0xc, 0xc420051b17, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc420051ac8 sp=0xc420051a98 pc=0x102fa7c
runtime.goparkunlock(0xc420074418, 0x1308f6e, 0xc, 0xc4200e8017, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc420051b08 sp=0xc420051ac8 pc=0x102fb6e
runtime.chanrecv(0xc4200743c0, 0x0, 0xc420051b01, 0x10ebc14)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc420051bb8 sp=0xc420051b08 pc=0x1005dc4
runtime.chanrecv1(0xc4200743c0, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc420051be8 sp=0xc420051bb8 pc=0x1005a6b
testing.(*T).Run(0xc4200e2000, 0x130879c, 0xb, 0x131b2e0, 0x108f201)
	/Users/hiro/go/src/testing/testing.go:801 +0x332 fp=0xc420051c98 sp=0xc420051be8 pc=0x10ebc32
testing.runTests.func1(0xc4200e2000)
	/Users/hiro/go/src/testing/testing.go:1053 +0x64 fp=0xc420051ce8 sp=0xc420051c98 pc=0x10efd04
testing.tRunner(0xc4200e2000, 0xc420051d98)
	/Users/hiro/go/src/testing/testing.go:754 +0xd0 fp=0xc420051d10 sp=0xc420051ce8 pc=0x10eb8a0
testing.runTests(0xc42000c300, 0x1498000, 0xd7, 0xd7, 0x5f)
	/Users/hiro/go/src/testing/testing.go:1051 +0x2d8 fp=0xc420051dc8 sp=0xc420051d10 pc=0x10ed688
testing.(*M).Run(0xc420051f18, 0x12474b6)
	/Users/hiro/go/src/testing/testing.go:932 +0x111 fp=0xc420051ec0 sp=0xc420051dc8 pc=0x10ec031
runtime_test.TestMain(0xc420051f18)
	/Users/hiro/go/src/runtime/crash_test.go:28 +0x2f fp=0xc420051f10 sp=0xc420051ec0 pc=0x1216c9f
main.main()
	runtime/_test/_testmain.go:902 +0xdb fp=0xc420051f80 sp=0xc420051f10 pc=0x126701b
runtime.main()
	/Users/hiro/go/src/runtime/proc.go:185 +0x20d fp=0xc420051fe0 sp=0xc420051f80 pc=0x102f5dd
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420051fe8 sp=0xc420051fe0 pc=0x10612f1

goroutine 2 [force gc (idle)]:
runtime.gopark(0x131a290, 0x149a0c0, 0x130a49c, 0xf, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002c768 sp=0xc42002c738 pc=0x102fa7c
runtime.goparkunlock(0x149a0c0, 0x130a49c, 0xf, 0xc420000114, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002c7a8 sp=0xc42002c768 pc=0x102fb6e
runtime.forcegchelper()
	/Users/hiro/go/src/runtime/proc.go:235 +0xcc fp=0xc42002c7e0 sp=0xc42002c7a8 pc=0x102f89c
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002c7e8 sp=0xc42002c7e0 pc=0x10612f1
created by runtime.init.4
	/Users/hiro/go/src/runtime/proc.go:224 +0x35

goroutine 3 [GC sweep wait]:
runtime.gopark(0x131a290, 0x149a3c0, 0x130941f, 0xd, 0x1021014, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002cf60 sp=0xc42002cf30 pc=0x102fa7c
runtime.goparkunlock(0x149a3c0, 0x130941f, 0xd, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002cfa0 sp=0xc42002cf60 pc=0x102fb6e
runtime.bgsweep(0xc420020070)
	/Users/hiro/go/src/runtime/mgcsweep.go:52 +0xa3 fp=0xc42002cfd8 sp=0xc42002cfa0 pc=0x10210d3
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002cfe0 sp=0xc42002cfd8 pc=0x10612f1
created by runtime.gcenable
	/Users/hiro/go/src/runtime/mgc.go:216 +0x58

goroutine 4 [finalizer wait]:
runtime.gopark(0x131a290, 0x14b9f10, 0x1309da5, 0xe, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002d700 sp=0xc42002d6d0 pc=0x102fa7c
runtime.goparkunlock(0x14b9f10, 0x1309da5, 0xe, 0x14, 0x1)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002d740 sp=0xc42002d700 pc=0x102fb6e
runtime.runfinq()
	/Users/hiro/go/src/runtime/mfinal.go:175 +0xb8 fp=0xc42002d7e0 sp=0xc42002d740 pc=0x1017ed8
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002d7e8 sp=0xc42002d7e0 pc=0x10612f1
created by runtime.createfing
	/Users/hiro/go/src/runtime/mfinal.go:156 +0x62

goroutine 5 [syscall]:
runtime.notetsleepg(0x14ba2c0, 0xffffffffffffffff, 0x0)
	/Users/hiro/go/src/runtime/lock_sema.go:280 +0x4b fp=0xc42002df80 sp=0xc42002df40 pc=0x10119eb
os/signal.signal_recv(0x0)
	/Users/hiro/go/src/runtime/sigqueue.go:131 +0xa7 fp=0xc42002dfa8 sp=0xc42002df80 pc=0x1045807
os/signal.loop()
	/Users/hiro/go/src/os/signal/signal_unix.go:22 +0x22 fp=0xc42002dfe0 sp=0xc42002dfa8 pc=0x10e4232
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002dfe8 sp=0xc42002dfe0 pc=0x10612f1
created by os/signal.init.0
	/Users/hiro/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 6 [select, locked to thread]:
runtime.gopark(0x131a2d8, 0x0, 0x1306abf, 0x6, 0x18, 0x1)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42003aca0 sp=0xc42003ac70 pc=0x102fa7c
runtime.selectgo(0xc42003af50, 0xc420074180)
	/Users/hiro/go/src/runtime/select.go:395 +0x1138 fp=0xc42003af18 sp=0xc42003aca0 pc=0x103fff8
runtime.ensureSigM.func1()
	/Users/hiro/go/src/runtime/signal_unix.go:511 +0x1fe fp=0xc42003afe0 sp=0xc42003af18 pc=0x105da9e
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42003afe8 sp=0xc42003afe0 pc=0x10612f1
created by runtime.ensureSigM
	/Users/hiro/go/src/runtime/signal_unix.go:494 +0xda

goroutine 7 [chan receive]:
runtime.gopark(0x131a290, 0xc42005e358, 0x1308f6e, 0xc, 0xc420024117, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002ee80 sp=0xc42002ee50 pc=0x102fa7c
runtime.goparkunlock(0xc42005e358, 0x1308f6e, 0xc, 0x17, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002eec0 sp=0xc42002ee80 pc=0x102fb6e
runtime.chanrecv(0xc42005e300, 0x0, 0x1, 0x0)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc42002ef70 sp=0xc42002eec0 pc=0x1005dc4
runtime.chanrecv1(0xc42005e300, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc42002efa0 sp=0xc42002ef70 pc=0x1005a6b
testing.(*M).before.func1(0xc42005e300)
	/Users/hiro/go/src/testing/testing.go:1111 +0x38 fp=0xc42002efd8 sp=0xc42002efa0 pc=0x10efd98
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002efe0 sp=0xc42002efd8 pc=0x10612f1
created by testing.(*M).before
	/Users/hiro/go/src/testing/testing.go:1110 +0x191

goroutine 8 [chan receive]:
runtime.gopark(0x131a290, 0xc420106058, 0x1308f6e, 0xc, 0xc42002f617, 0x3)
	/Users/hiro/go/src/runtime/proc.go:277 +0x12c fp=0xc42002f5e8 sp=0xc42002f5b8 pc=0x102fa7c
runtime.goparkunlock(0xc420106058, 0x1308f6e, 0xc, 0x1010f0120104017, 0x3)
	/Users/hiro/go/src/runtime/proc.go:283 +0x5e fp=0xc42002f628 sp=0xc42002f5e8 pc=0x102fb6e
runtime.chanrecv(0xc420106000, 0x0, 0xc42002f701, 0x1249879)
	/Users/hiro/go/src/runtime/chan.go:506 +0x304 fp=0xc42002f6d8 sp=0xc42002f628 pc=0x1005dc4
runtime.chanrecv1(0xc420106000, 0x0)
	/Users/hiro/go/src/runtime/chan.go:388 +0x2b fp=0xc42002f708 sp=0xc42002f6d8 pc=0x1005a6b
runtime_test.HammerRWMutex(0x4, 0xa, 0x3e8)
	/Users/hiro/go/src/runtime/rwmutex_test.go:103 +0x1eb fp=0xc42002f770 sp=0xc42002f708 pc=0x12498ab
runtime_test.TestRWMutex(0xc4200e20f0)
	/Users/hiro/go/src/runtime/rwmutex_test.go:118 +0x12f fp=0xc42002f7a8 sp=0xc42002f770 pc=0x1249a0f
testing.tRunner(0xc4200e20f0, 0x131b2e0)
	/Users/hiro/go/src/testing/testing.go:754 +0xd0 fp=0xc42002f7d0 sp=0xc42002f7a8 pc=0x10eb8a0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42002f7d8 sp=0xc42002f7d0 pc=0x10612f1
created by testing.(*T).Run
	/Users/hiro/go/src/testing/testing.go:800 +0x314

goroutine 51 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 50 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:92 +0xc1

goroutine 52 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010efc0 sp=0xc42010efb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010efc8 sp=0xc42010efc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 53 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010f7c0 sp=0xc42010f7b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010f7c8 sp=0xc42010f7c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 54 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 55 [running]:
	goroutine running on other thread; stack unavailable
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105

goroutine 56 [runnable]:
runtime_test.writer(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:71 fp=0xc420110fc0 sp=0xc420110fb8 pc=0x1249540
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420110fc8 sp=0xc420110fc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:97 +0x16d

goroutine 57 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc4201117c0 sp=0xc4201117b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4201117c8 sp=0xc4201117c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 58 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc420111fc0 sp=0xc420111fb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420111fc8 sp=0xc420111fc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 59 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc4200f47c0 sp=0xc4200f47b8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4200f47c8 sp=0xc4200f47c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 60 [runnable]:
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:56 fp=0xc42010afc0 sp=0xc42010afb8 pc=0x12493c0
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010afc8 sp=0xc42010afc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

goroutine 61 [runnable]:
runtime.(*RWMutex).RUnlock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:361 +0x3c fp=0xc42010b748 sp=0xc42010b740 pc=0x105a9bc
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:66 +0x45 fp=0xc42010b7c0 sp=0xc42010b748 pc=0x1249405
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010b7c8 sp=0xc42010b7c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:99 +0x1b9

rax    0x4
rbx    0x3
rcx    0x700005e8ddc0
rdx    0x0
rdi    0x0
rsi    0x0
rbp    0x700005e8ddd0
rsp    0x700005e8ddc0
r8     0x700005e8ddc0
r9     0x16
r10    0x0
r11    0x246
r12    0x746f44e68d77
r13    0xb03
r14    0x10326d0
r15    0x1602620
rip    0x10625b6
rflags 0x247
cs     0x7
fs     0x0
gs     0x0

-----

SIGQUIT: quit
PC=0x10626db m=3 sigcode=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0xc400001403, 0x103adeb, 0xc420023300, 0x149a001, 0x700005f10e38, 0xc420000d80, 0x700005f10e50, 0x105c6b3, 0xffffffffffffffff, 0x1036752, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:445 +0xb fp=0x700005f10df8 sp=0x700005f10df0 pc=0x10626db
runtime.semasleep1(0xffffffffffffffff, 0x1036752)
	/Users/hiro/go/src/runtime/os_darwin.go:413 +0x52 fp=0x700005f10e38 sp=0x700005f10df8 pc=0x102bc52
runtime.semasleep.func1()
	/Users/hiro/go/src/runtime/os_darwin.go:432 +0x33 fp=0x700005f10e60 sp=0x700005f10e38 pc=0x105c6b3
runtime.systemstack(0xc420023300)
	/Users/hiro/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x700005f10e68 sp=0x700005f10e60 pc=0x105e769
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1125 fp=0x700005f10e70 sp=0x700005f10e68 pc=0x10326d0
rax    0xe
rbx    0x700005f10e60
rcx    0x700005f10df0
rdx    0xc42002dee8
rdi    0x1403
rsi    0x105e6e0
rbp    0x700005f10e28
rsp    0x700005f10df0
r8     0xc420001500
r9     0xc420000d80
r10    0x0
r11    0x286
r12    0x0
r13    0xf1
r14    0x11
r15    0x0
rip    0x10626db
rflags 0x286
cs     0x7
fs     0x0
gs     0x0

-----

SIGQUIT: quit
PC=0x10626db m=4 sigcode=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0x700000001103, 0x0, 0xc420068110, 0x1, 0xc420025900, 0xc420001200, 0x700005f93db0, 0x105c6b3, 0xffffffffffffffff, 0x0, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:445 +0xb fp=0x700005f93d58 sp=0x700005f93d50 pc=0x10626db
runtime.semasleep1(0xffffffffffffffff, 0x0)
	/Users/hiro/go/src/runtime/os_darwin.go:413 +0x52 fp=0x700005f93d98 sp=0x700005f93d58 pc=0x102bc52
runtime.semasleep.func1()
	/Users/hiro/go/src/runtime/os_darwin.go:432 +0x33 fp=0x700005f93dc0 sp=0x700005f93d98 pc=0x105c6b3
runtime.systemstack(0x700005f93dd8)
	/Users/hiro/go/src/runtime/asm_amd64.s:360 +0xab fp=0x700005f93dc8 sp=0x700005f93dc0 pc=0x105e79b
runtime.semasleep(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:431 +0x44 fp=0x700005f93e00 sp=0x700005f93dc8 pc=0x102bd84
runtime.notesleep(0xc420030b90)
	/Users/hiro/go/src/runtime/lock_sema.go:167 +0xe9 fp=0x700005f93e38 sp=0x700005f93e00 pc=0x1011609
runtime.(*rwmutex).lock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:89 +0x5c fp=0x700005f93e60 sp=0x700005f93e38 pc=0x105d71c
runtime.systemstack(0xc420022000)
	/Users/hiro/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x700005f93e68 sp=0x700005f93e60 pc=0x105e769
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1125 fp=0x700005f93e70 sp=0x700005f93e68 pc=0x10326d0
goroutine 50 [running]:
runtime.systemstack_switch()
	/Users/hiro/go/src/runtime/asm_amd64.s:298 fp=0xc4200f56f0 sp=0xc4200f56e8 pc=0x105e6e0
runtime.(*rwmutex).lock(0xc420104000)
	/Users/hiro/go/src/runtime/rwmutex.go:86 +0xc3 fp=0xc4200f5730 sp=0xc4200f56f0 pc=0x103e623
runtime.(*RWMutex).Lock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:366 +0x2d fp=0xc4200f5748 sp=0xc4200f5730 pc=0x105a9ed
runtime_test.writer(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:73 +0x6d fp=0xc4200f57c0 sp=0xc4200f5748 pc=0x12495ad
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4200f57c8 sp=0xc4200f57c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:92 +0xc1
rax    0xe
rbx    0xc420030a80
rcx    0x700005f93d50
rdx    0x700005f93dd8
rdi    0x1103
rsi    0x105e601
rbp    0x700005f93d88
rsp    0x700005f93d50
r8     0xc42006a480
r9     0x15
r10    0x0
r11    0x286
r12    0xc420100038
r13    0xf1
r14    0x11
r15    0x0
rip    0x10626db
rflags 0x286
cs     0x7
fs     0x0
gs     0x0

-----

SIGQUIT: quit
PC=0x10626db m=5 sigcode=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0xc400001603, 0x700000000000, 0x10339a1, 0xc4200c6110, 0x1, 0xc42006a180, 0x700006016da8, 0x105c6b3, 0xffffffffffffffff, 0xc400000000, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:445 +0xb fp=0x700006016d50 sp=0x700006016d48 pc=0x10626db
runtime.semasleep1(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:413 +0x52 fp=0x700006016d90 sp=0x700006016d50 pc=0x102bc52
runtime.semasleep.func1()
	/Users/hiro/go/src/runtime/os_darwin.go:432 +0x33 fp=0x700006016db8 sp=0x700006016d90 pc=0x105c6b3
runtime.systemstack(0x700006016dd0)
	/Users/hiro/go/src/runtime/asm_amd64.s:360 +0xab fp=0x700006016dc0 sp=0x700006016db8 pc=0x105e79b
runtime.semasleep(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:431 +0x44 fp=0x700006016df8 sp=0x700006016dc0 pc=0x102bd84
runtime.notesleep(0xc420068110)
	/Users/hiro/go/src/runtime/lock_sema.go:167 +0xe9 fp=0x700006016e30 sp=0x700006016df8 pc=0x1011609
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x700006016e60 sp=0x700006016e30 pc=0x105d696
runtime.systemstack(0xc420024600)
	/Users/hiro/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x700006016e68 sp=0x700006016e60 pc=0x105e769
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1125 fp=0x700006016e70 sp=0x700006016e68 pc=0x10326d0
goroutine 51 [running]:
runtime.systemstack_switch()
	/Users/hiro/go/src/runtime/asm_amd64.s:298 fp=0xc4200f9708 sp=0xc4200f9700 pc=0x105e6e0
runtime.(*rwmutex).rlock(0xc420104000)
	/Users/hiro/go/src/runtime/rwmutex.go:36 +0x5b fp=0xc4200f9730 sp=0xc4200f9708 pc=0x103e48b
runtime.(*RWMutex).RLock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:358 +0x2d fp=0xc4200f9748 sp=0xc4200f9730 pc=0x105a96d
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:58 +0x6d fp=0xc4200f97c0 sp=0xc4200f9748 pc=0x124942d
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4200f97c8 sp=0xc4200f97c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105
rax    0xe
rbx    0xc420068000
rcx    0x700006016d48
rdx    0x700006016dd0
rdi    0x1603
rsi    0x105e601
rbp    0x700006016d80
rsp    0x700006016d48
r8     0xc4200c9080
r9     0xc42006a180
r10    0x0
r11    0x286
r12    0xc420100038
r13    0xf1
r14    0x11
r15    0x0
rip    0x10626db
rflags 0x286
cs     0x7
fs     0x0
gs     0x0

-----

SIGQUIT: quit
PC=0x10626db m=6 sigcode=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0xc400001903, 0x0, 0xc420108300, 0x100700006099d88, 0x3, 0xc4200c8180, 0x700006099da8, 0x105c6b3, 0xffffffffffffffff, 0xc400000000, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:445 +0xb fp=0x700006099d50 sp=0x700006099d48 pc=0x10626db
runtime.semasleep1(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:413 +0x52 fp=0x700006099d90 sp=0x700006099d50 pc=0x102bc52
runtime.semasleep.func1()
	/Users/hiro/go/src/runtime/os_darwin.go:432 +0x33 fp=0x700006099db8 sp=0x700006099d90 pc=0x105c6b3
runtime.systemstack(0x700006099dd0)
	/Users/hiro/go/src/runtime/asm_amd64.s:360 +0xab fp=0x700006099dc0 sp=0x700006099db8 pc=0x105e79b
runtime.semasleep(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:431 +0x44 fp=0x700006099df8 sp=0x700006099dc0 pc=0x102bd84
runtime.notesleep(0xc4200c6110)
	/Users/hiro/go/src/runtime/lock_sema.go:167 +0xe9 fp=0x700006099e30 sp=0x700006099df8 pc=0x1011609
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x700006099e60 sp=0x700006099e30 pc=0x105d696
runtime.systemstack(0xc420023300)
	/Users/hiro/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x700006099e68 sp=0x700006099e60 pc=0x105e769
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1125 fp=0x700006099e70 sp=0x700006099e68 pc=0x10326d0
goroutine 54 [running]:
runtime.systemstack_switch()
	/Users/hiro/go/src/runtime/asm_amd64.s:298 fp=0xc42010ff08 sp=0xc42010ff00 pc=0x105e6e0
runtime.(*rwmutex).rlock(0xc420104000)
	/Users/hiro/go/src/runtime/rwmutex.go:36 +0x5b fp=0xc42010ff30 sp=0xc42010ff08 pc=0x103e48b
runtime.(*RWMutex).RLock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:358 +0x2d fp=0xc42010ff48 sp=0xc42010ff30 pc=0x105a96d
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:58 +0x6d fp=0xc42010ffc0 sp=0xc42010ff48 pc=0x124942d
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42010ffc8 sp=0xc42010ffc0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105
rax    0xe
rbx    0xc4200c6000
rcx    0x700006099d48
rdx    0x700006099dd0
rdi    0x1903
rsi    0x105e601
rbp    0x700006099d80
rsp    0x700006099d48
r8     0xc420108300
r9     0x6
r10    0xc420100040
r11    0x286
r12    0xc420100038
r13    0xf1
r14    0x11
r15    0x0
rip    0x10626db
rflags 0x286
cs     0x7
fs     0x0
gs     0x0

-----

SIGQUIT: quit
PC=0x10626db m=7 sigcode=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0x1c03, 0x0, 0x70000611cd78, 0xfffffffffffffade, 0xc420001c80, 0xc420001c80, 0x70000611cda8, 0x105c6b3, 0xffffffffffffffff, 0xc400000000, ...)
	/Users/hiro/go/src/runtime/sys_darwin_amd64.s:445 +0xb fp=0x70000611cd50 sp=0x70000611cd48 pc=0x10626db
runtime.semasleep1(0xffffffffffffffff, 0xc400000000)
	/Users/hiro/go/src/runtime/os_darwin.go:413 +0x52 fp=0x70000611cd90 sp=0x70000611cd50 pc=0x102bc52
runtime.semasleep.func1()
	/Users/hiro/go/src/runtime/os_darwin.go:432 +0x33 fp=0x70000611cdb8 sp=0x70000611cd90 pc=0x105c6b3
runtime.systemstack(0x70000611cdd0)
	/Users/hiro/go/src/runtime/asm_amd64.s:360 +0xab fp=0x70000611cdc0 sp=0x70000611cdb8 pc=0x105e79b
runtime.semasleep(0xffffffffffffffff, 0x700000000000)
	/Users/hiro/go/src/runtime/os_darwin.go:431 +0x44 fp=0x70000611cdf8 sp=0x70000611cdc0 pc=0x102bd84
runtime.notesleep(0xc420031d10)
	/Users/hiro/go/src/runtime/lock_sema.go:167 +0xe9 fp=0x70000611ce30 sp=0x70000611cdf8 pc=0x1011609
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x70000611ce60 sp=0x70000611ce30 pc=0x105d696
runtime.systemstack(0xc420001cb8)
	/Users/hiro/go/src/runtime/asm_amd64.s:344 +0x79 fp=0x70000611ce68 sp=0x70000611ce60 pc=0x105e769
runtime.mstart()
	/Users/hiro/go/src/runtime/proc.go:1125 fp=0x70000611ce70 sp=0x70000611ce68 pc=0x10326d0
goroutine 55 [running]:
runtime.systemstack_switch()
	/Users/hiro/go/src/runtime/asm_amd64.s:298 fp=0xc420110708 sp=0xc420110700 pc=0x105e6e0
runtime.(*rwmutex).rlock(0xc420104000)
	/Users/hiro/go/src/runtime/rwmutex.go:36 +0x5b fp=0xc420110730 sp=0xc420110708 pc=0x103e48b
runtime.(*RWMutex).RLock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:358 +0x2d fp=0xc420110748 sp=0xc420110730 pc=0x105a96d
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:58 +0x6d fp=0xc4201107c0 sp=0xc420110748 pc=0x124942d
runtime.goexit()
	/Users/hiro/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc4201107c8 sp=0xc4201107c0 pc=0x10612f1
created by runtime_test.HammerRWMutex
	/Users/hiro/go/src/runtime/rwmutex_test.go:95 +0x105
rax    0xe
rbx    0xc420031c00
rcx    0x70000611cd48
rdx    0x70000611cdd0
rdi    0x1c03
rsi    0x105e601
rbp    0x70000611cd80
rsp    0x70000611cd48
r8     0xc420108480
r9     0xc420001a08
r10    0xc420106040
r11    0x286
r12    0xc420106038
r13    0xf1
r14    0x11
r15    0x0
rip    0x10626db
rflags 0x286
cs     0x7
fs     0x0
gs     0x0

-----

*** Test killed: ran too long (10m0s).
FAIL	runtime	605.013s

Thank you.

@aclements

This comment has been minimized.

Copy link
Member

commented Jul 6, 2017

It's only 4.

Oh, the test adjusts GOMAXPROCS itself. Sorry, should have thought of that.

Can you reproduce it with GOTRACEBACK=crash set and post the output?

goroutine 8 [chan receive]:
runtime_test.HammerRWMutex(0x4, 0xa, 0x3e8)

GOMAXPROCS=4, numReaders=10, n=1000

goroutine 52 [runnable]:
goroutine 53 [runnable]:
goroutine 57 [runnable]:
goroutine 58 [runnable]:
goroutine 59 [runnable]:
goroutine 60 [runnable]:

Readers that haven't been scheduled yet.

goroutine 56 [runnable]:

Writer that hasn't been scheduled yet.

goroutine 61 [runnable]:
runtime.(*RWMutex).RUnlock(0xc420104000)
	/Users/hiro/go/src/runtime/export_test.go:361 +0x3c fp=0xc42010b748 sp=0xc42010b740 pc=0x105a9bc
runtime_test.reader(0xc420104000, 0x3e8, 0xc420102000, 0xc420106000)
	/Users/hiro/go/src/runtime/rwmutex_test.go:66 +0x45 fp=0xc42010b7c0 sp=0xc42010b748 pc=0x1249405

This goroutine holds a read lock and was about to release it but got descheduled. This is probably the bug.

PC=0x10626db m=4 sigcode=0
runtime.(*rwmutex).lock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:89 +0x5c fp=0x700005f93e60 sp=0x700005f93e38 pc=0x105d71c
goroutine 50 [running]:

Pending writer, waiting to be woken by the last reader.

PC=0x10626db m=5 sigcode=0
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x700006016e60 sp=0x700006016e30 pc=0x105d696
goroutine 51 [running]:

Pending reader, on queue waiting to be woken after the writer releases.

PC=0x10626db m=6 sigcode=0
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x700006099e60 sp=0x700006099e30 pc=0x105d696
goroutine 54 [running]:

Same.

PC=0x10626db m=7 sigcode=0
runtime.(*rwmutex).rlock.func1()
	/Users/hiro/go/src/runtime/rwmutex.go:49 +0xa6 fp=0x70000611ce60 sp=0x70000611ce30 pc=0x105d696
goroutine 55 [running]:

Same.

So, the lock is in read mode, held by only goroutine 61. There's a pending writer and a few pending readers behind the writer. But goroutine 61 got descheduled, and meanwhile four other goroutines got scheduled and blocked their Ms and Ps attempting to acquire the lock. That consumed all of GOMAXPROCS, so goroutine 61 can't run and release its read lock, leading to deadlock.

Sounds like the read lock either needs to disallow rescheduling, or we need to release the P while blocking the M.

@aclements

This comment has been minimized.

Copy link
Member

commented Jul 6, 2017

I can now reproduce this easily by adding a Gosched in the middle of the reader loop.

Dropping the P is really messy since that requires using lockOSThread and notetsleepg, which winds up in stoplockedm, which itself uses m.park, causing things to go haywire when we try to wake a pending reader.

Disallowing rescheduling is less messy, though TestParallelRWMutexReaders is going to need some rewriting since it depends on potentially blocking channel sends.

@gopherbot

This comment has been minimized.

Copy link

commented Jul 6, 2017

CL https://golang.org/cl/47632 mentions this issue.

@aclements aclements modified the milestones: Go1.9, Go1.9Maybe Jul 6, 2017

@gopherbot gopherbot closed this in f3b5a2b Jul 6, 2017

@hirochachacha

This comment has been minimized.

Copy link
Contributor Author

commented Jul 6, 2017

Thank you for the fix, it works fine.
Also thank you for writing up the description, I'm glad to see that.

@aclements

This comment has been minimized.

Copy link
Member

commented Jul 6, 2017

Thank you for reporting the issue. :)

@golang golang locked and limited conversation to collaborators Jul 6, 2018

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
You can’t perform that action at this time.