Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: corruption in tests #55167

Closed
rsc opened this issue Sep 20, 2022 · 5 comments
Closed

runtime: corruption in tests #55167

rsc opened this issue Sep 20, 2022 · 5 comments
Assignees
Labels
compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one.
Milestone

Comments

@rsc
Copy link
Contributor

rsc commented Sep 20, 2022

#!watchflakes
default <- `runtime.throw\(` || `fatal error:`

This issue gathers flakes caused by runtime.throw, which should "never happen" and usually indicates some kind of corruption.

@rsc rsc added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label Sep 20, 2022
@gopherbot gopherbot added the compiler/runtime Issues related to the Go compiler and/or runtime. label Sep 20, 2022
@gopherbot
Copy link

gopherbot commented Sep 20, 2022

Found new matching flaky dashboard failures for:

#!watchflakes
post <- `runtime.throw\(` || `fatal error:`
2022-08-08 14:11 windows-386-2012 go@6e9925c4 runtime (log)
Exception 0xc7 0xcccccccc 0xcccccccc 0x0
PC=0x0

fatal error: unexpected signal during runtime execution
[signal 0xc0000005 code=0x0 addr=0x0 pc=0x465d7a]

runtime stack:
runtime.throw({0x7f14e7, 0x2a})
	C:/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x328ffbdc sp=0x328ffbc8 pc=0x44047d
runtime.sigpanic()
...
fatal error: unexpected signal during runtime execution
panic during panic
[signal 0xc0000005 code=0x0 addr=0x0 pc=0x46631f]

runtime stack:
runtime.throw({0x7f14e7, 0x2a})
	C:/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x328ff824 sp=0x328ff810 pc=0x44047d
runtime.sigpanic()
	C:/workdir/go/src/runtime/signal_windows.go:250 +0x23b fp=0x328ff848 sp=0x328ff824 pc=0x455fab
runtime.gentraceback(0xffffffff, 0xffffffff, 0x0, 0x142ca4b0, 0x0, 0x0, 0x64, 0x0, 0x0, 0x0)
...
	C:/workdir/go/src/runtime/traceback.go:846 +0x1a2 fp=0x328ffef0 sp=0x328ffe10 pc=0x468562
runtime.tracebacktrap(0x0, 0x0, 0x0, 0x142ca4b0)
	C:/workdir/go/src/runtime/traceback.go:810 +0x7e fp=0x328fff08 sp=0x328ffef0 pc=0x4683ae
runtime.winthrow(0x750df4, 0x12860ad0, 0x142ca4b0)
	C:/workdir/go/src/runtime/signal_windows.go:235 +0x202 fp=0x328fff34 sp=0x328fff08 pc=0x4559b2
runtime.lastcontinuehandler(0x750df4, 0x12860ad0, 0x142ca4b0)
	C:/workdir/go/src/runtime/signal_windows.go:198 +0x58 fp=0x328fff44 sp=0x328fff34 pc=0x455798
sigtramp()
	C:/workdir/go/src/runtime/sys_windows_386.s:127 +0x77 fp=0x328fff48 sp=0x328fff44 pc=0x477657
*** Test killed: ran too long (6m0s).
2022-08-08 16:57 js-wasm go@b589208c go/internal/gcimporter (log)
runtime: marked free object in span 0x369ed8, elemsize=5376 freeindex=0 (bad use of unsafe.Pointer? try -d=checkptr)
0x456000 alloc marked  
0x457500 free  unmarked
0x458a00 free  marked   zombie
0x0000000000458a00:  0x544f4f524f472420  0x2f6f672f6372732f 
0x0000000000458a10:  0x2f72656e6e616373  0x672e73726f727265 
0x0000000000458a20:  0x5008736f5003006f  0x086e6f697469736f 
0x0000000000458a30:  0x6e656b6f742f6f67  0x7272450567734d03 
0x0000000000458a40:  0x637365066501726f  0x016464410301003a 
0x0000000000458a50:  0x4c726f7272450970  0x03736f7003747369 
...
fatal error: found pointer to free object

goroutine 3 [running]:
runtime.throw({0x7d89b, 0x1c})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x4315e0 sp=0x4315b8 pc=0x12460003
runtime.(*mspan).reportZombies(0x369ed8)
	/workdir/go/src/runtime/mgcsweep.go:788 +0x60 fp=0x431660 sp=0x4315e0 pc=0x118b0060
runtime.(*sweepLocked).sweep(0x431780, 0x0)
	/workdir/go/src/runtime/mgcsweep.go:607 +0x5f fp=0x431748 sp=0x431660 pc=0x118a005f
runtime.sweepone()
	/workdir/go/src/runtime/mgcsweep.go:370 +0x18 fp=0x4317a8 sp=0x431748 pc=0x11880018
runtime.bgsweep(0x438000)
	/workdir/go/src/runtime/mgcsweep.go:282 +0x2c fp=0x4317d0 sp=0x4317a8 pc=0x1185002c
runtime.gcenable.func1()
	/workdir/go/src/runtime/mgc.go:178 +0x2 fp=0x4317e0 sp=0x4317d0 pc=0x110a0002
runtime.goexit()
	/workdir/go/src/runtime/asm_wasm.s:401 +0x1 fp=0x4317e8 sp=0x4317e0 pc=0x14350001
created by runtime.gcenable
	/workdir/go/src/runtime/mgc.go:178 +0x8
2022-08-08 16:58 js-wasm go@2493072d go/build (log)
runtime: marked free object in span 0xd7ef80, elemsize=288 freeindex=4 (bad use of unsafe.Pointer? try -d=checkptr)
0x676000 alloc unmarked
0x676120 alloc marked  
0x676240 alloc unmarked
0x676360 alloc unmarked
0x676480 free  unmarked
0x6765a0 free  marked   zombie
0x00000000006765a0:  0x0000000000000000  0x0000000000000000 
0x00000000006765b0:  0x0000000000000000  0x0000000000000000 
0x00000000006765c0:  0x0000000000000000  0x0000000000000000 
...
fatal error: found pointer to free object

runtime stack:
runtime.throw({0x558d1, 0x1c})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x265d88 sp=0x265d60 pc=0x12430003
runtime.(*mspan).reportZombies(0xd7ef80)
	/workdir/go/src/runtime/mgcsweep.go:788 +0x60 fp=0x265e08 sp=0x265d88 pc=0x11880060
runtime.(*sweepLocked).sweep(0x265f10, 0x0)
	/workdir/go/src/runtime/mgcsweep.go:607 +0x5f fp=0x265ef0 sp=0x265e08 pc=0x1187005f
runtime.(*mcentral).uncacheSpan(0x24ccd0, 0xd7ef80)
...
runtime.gcMarkTermination()
	/workdir/go/src/runtime/mgc.go:1058 +0x6c fp=0x432f00 sp=0x432cf8 pc=0x1118006c
runtime.gcMarkDone()
	/workdir/go/src/runtime/mgc.go:918 +0x45 fp=0x432f58 sp=0x432f00 pc=0x11120045
runtime.gcBgMarkWorker()
	/workdir/go/src/runtime/mgc.go:1366 +0x66 fp=0x432fe0 sp=0x432f58 pc=0x111c0066
runtime.goexit()
	/workdir/go/src/runtime/asm_wasm.s:401 +0x1 fp=0x432fe8 sp=0x432fe0 pc=0x14360001
created by runtime.gcBgMarkStartWorkers
	/workdir/go/src/runtime/mgc.go:1158 +0x3
2022-08-08 17:36 linux-386-longtest go@ebf182c8 runtime/pprof (log)
runtime: pointer 0x616e6966 to unallocated span span.base()=0x8f78000 span.limit=0x8f79ff8 span.state=0
runtime: found in object at *(0x8d3a420+0x0)
object=0x8d3a420 s.base()=0x8d3a000 s.limit=0x8d3c000 s.spanclass=5 s.elemsize=16 s.state=mSpanInUse
 *(object+0) = 0x616e6966 <==
 *(object+4) = 0x657a696c
 *(object+8) = 0x8114ed7
 *(object+12) = 0x0
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x81d8da0, 0x3e})
	/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x8c53e2c sp=0x8c53e18 pc=0x807fbdd
runtime.badPointer(0xef804360, 0x616e6966, 0x8d3a420, 0x0)
	/workdir/go/src/runtime/mbitmap.go:316 +0xa2 fp=0x8c53e4c sp=0x8c53e2c pc=0x8058f12
runtime.findObject(0x616e6966, 0x8d3a420, 0x0)
	/workdir/go/src/runtime/mbitmap.go:359 +0x9e fp=0x8c53e64 sp=0x8c53e4c pc=0x805916e
runtime.scanobject(0x8d3a420, 0x8c2895c)
	/workdir/go/src/runtime/mgcmark.go:1341 +0x183 fp=0x8c53eb4 sp=0x8c53e64 pc=0x8065613
runtime.markrootSpans(0x8c2895c, 0x0)
	/workdir/go/src/runtime/mgcmark.go:390 +0x18e fp=0x8c53ef0 sp=0x8c53eb4 pc=0x80630fe
runtime.markroot(0x8c2895c, 0x4, 0x1)
	/workdir/go/src/runtime/mgcmark.go:192 +0xe9 fp=0x8c53f4c sp=0x8c53ef0 pc=0x80628c9
runtime.gcDrain(0x8c2895c, 0x3)
	/workdir/go/src/runtime/mgcmark.go:1069 +0x4ed fp=0x8c53fb0 sp=0x8c53f4c pc=0x8064ffd
runtime.gcBgMarkWorker.func2()
	/workdir/go/src/runtime/mgc.go:1307 +0xaa fp=0x8c53fdc sp=0x8c53fb0 pc=0x806060a
runtime.systemstack()
	/workdir/go/src/runtime/asm_386.s:370 +0x41 fp=0x8c53fe0 sp=0x8c53fdc pc=0x80af4e1
2022-08-08 17:39 js-wasm go@d42b520b go/internal/gcimporter (log)
runtime: marked free object in span 0x369a10, elemsize=2304 freeindex=0 (bad use of unsafe.Pointer? try -d=checkptr)
0x46c000 alloc marked  
0x46c900 free  unmarked
0x46d200 free  unmarked
0x46db00 free  unmarked
0x46e400 free  marked   zombie
0x000000000046e400:  0x0000000000000000  0x0000000000000000 
0x000000000046e410:  0x0000000000000000  0x0000000000000000 
0x000000000046e420:  0x0000000000000000  0x0000000000000000 
0x000000000046e430:  0x0000000000000000  0x0000000000000000 
...
fatal error: found pointer to free object

goroutine 3 [running]:
runtime.throw({0x7d888, 0x1c})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x4315e0 sp=0x4315b8 pc=0x12460003
runtime.(*mspan).reportZombies(0x369a10)
	/workdir/go/src/runtime/mgcsweep.go:788 +0x60 fp=0x431660 sp=0x4315e0 pc=0x118b0060
runtime.(*sweepLocked).sweep(0x431780, 0x0)
	/workdir/go/src/runtime/mgcsweep.go:607 +0x5f fp=0x431748 sp=0x431660 pc=0x118a005f
runtime.sweepone()
	/workdir/go/src/runtime/mgcsweep.go:370 +0x18 fp=0x4317a8 sp=0x431748 pc=0x11880018
runtime.bgsweep(0x438000)
	/workdir/go/src/runtime/mgcsweep.go:282 +0x2c fp=0x4317d0 sp=0x4317a8 pc=0x1185002c
runtime.gcenable.func1()
	/workdir/go/src/runtime/mgc.go:178 +0x2 fp=0x4317e0 sp=0x4317d0 pc=0x110a0002
runtime.goexit()
	/workdir/go/src/runtime/asm_wasm.s:401 +0x1 fp=0x4317e8 sp=0x4317e0 pc=0x14350001
created by runtime.gcenable
	/workdir/go/src/runtime/mgc.go:178 +0x8
2022-08-08 17:39 js-wasm go@d02d5fda go/internal/gcimporter (log)
runtime: pointer 0x7e42a0 to unused region of span span.base()=0x522000 span.limit=0x526000 span.state=1
runtime: found in object at *(0x424f00+0x90)
object=0x424f00 s.base()=0x41e000 s.limit=0x42be00 s.spanclass=104 s.elemsize=9472 s.state=mSpanInUse
 *(object+0) = 0x8fd8
 *(object+8) = 0x6e63d0
 *(object+16) = 0xc
 *(object+24) = 0x6e67d0
 *(object+32) = 0xa
 *(object+40) = 0x0
 *(object+48) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x8604a, 0x3e})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x351220 sp=0x3511f8 pc=0x12460003
runtime.badPointer(0xdfb508, 0x7e42a0, 0x424f00, 0x90)
	/workdir/go/src/runtime/mbitmap.go:316 +0xe fp=0x351260 sp=0x351220 pc=0x10dc000e
runtime.findObject(0x7e42a0, 0x424f00, 0x90)
	/workdir/go/src/runtime/mbitmap.go:359 +0x1d fp=0x351290 sp=0x351260 pc=0x10dd001d
runtime.scanobject(0x424f00, 0x41f240)
	/workdir/go/src/runtime/mgcmark.go:1341 +0x32 fp=0x351328 sp=0x351290 pc=0x11460032
runtime.gcDrain(0x41f240, 0x7)
	/workdir/go/src/runtime/mgcmark.go:1103 +0x35 fp=0x351380 sp=0x351328 pc=0x11430035
runtime.gcBgMarkWorker.func2()
	/workdir/go/src/runtime/mgc.go:1326 +0x8 fp=0x3513d8 sp=0x351380 pc=0x11200008
runtime.systemstack()
	/workdir/go/src/runtime/asm_wasm.s:169 +0x2 fp=0x3513e0 sp=0x3513d8 pc=0x140c0002
runtime.mstart()
	/workdir/go/src/runtime/asm_wasm.s:27 fp=0x3513e8 sp=0x3513e0 pc=0x14080000
2022-08-09 14:33 js-wasm go@0981d9ff go/build (log)
runtime: pointer 0x737973 to unused region of span span.base()=0x498000 span.limit=0x49a000 span.state=1
runtime: found in object at *(0x61c560+0x8)
object=0x61c560 s.base()=0x61c000 s.limit=0x61e000 s.spanclass=5 s.elemsize=16 s.state=mSpanInUse
 *(object+0) = 0x7ff8000100001eb1
 *(object+8) = 0x737973 <==
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x5b8ee, 0x3e})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x265e00 sp=0x265dd8 pc=0x12430003
runtime.badPointer(0xcbd0a8, 0x737973, 0x61c560, 0x8)
	/workdir/go/src/runtime/mbitmap.go:316 +0xe fp=0x265e40 sp=0x265e00 pc=0x10d9000e
runtime.findObject(0x737973, 0x61c560, 0x8)
	/workdir/go/src/runtime/mbitmap.go:359 +0x1d fp=0x265e70 sp=0x265e40 pc=0x10da001d
runtime.scanobject(0x61c560, 0x41f240)
	/workdir/go/src/runtime/mgcmark.go:1341 +0x32 fp=0x265f08 sp=0x265e70 pc=0x11430032
runtime.markrootSpans(0x41f240, 0x0)
	/workdir/go/src/runtime/mgcmark.go:390 +0x34 fp=0x265f80 sp=0x265f08 pc=0x11360034
runtime.markroot(0x41f240, 0x4, 0x1)
	/workdir/go/src/runtime/mgcmark.go:192 +0x1d fp=0x266028 sp=0x265f80 pc=0x1132001d
runtime.gcDrain(0x41f240, 0xb)
	/workdir/go/src/runtime/mgcmark.go:1069 +0x66 fp=0x266080 sp=0x266028 pc=0x11400066
runtime.gcBgMarkWorker.func2()
	/workdir/go/src/runtime/mgc.go:1324 +0xa fp=0x2660d8 sp=0x266080 pc=0x111d000a
runtime.systemstack()
	/workdir/go/src/runtime/asm_wasm.s:169 +0x2 fp=0x2660e0 sp=0x2660d8 pc=0x140d0002
runtime.mstart()
	/workdir/go/src/runtime/asm_wasm.s:27 fp=0x2660e8 sp=0x2660e0 pc=0x14090000
2022-08-09 15:13 js-wasm go@0d9ed063 go/internal/gcimporter (log)
runtime: marked free object in span 0x36b968, elemsize=10240 freeindex=1 (bad use of unsafe.Pointer? try -d=checkptr)
0x4aa000 alloc unmarked
0x4ac800 free  unmarked
0x4af000 free  unmarked
0x4b1800 free  marked   zombie
0x00000000004b1800:  0x0000000000000012  0x0000000000000000 
0x00000000004b1810:  0x0000000000000000  0x0000000000000000 
0x00000000004b1820:  0x0000000000000000  0x0000000000000000 
0x00000000004b1830:  0x0000000000000000  0x0000000000000000 
0x00000000004b1840:  0x0000a3585de23a8c  0x0000000000002bac 
...
fatal error: found pointer to free object

runtime stack:
runtime.throw({0x7d885, 0x1c})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x351088 sp=0x351060 pc=0x12460003
runtime.(*mspan).reportZombies(0x36b968)
	/workdir/go/src/runtime/mgcsweep.go:788 +0x60 fp=0x351108 sp=0x351088 pc=0x118b0060
runtime.(*sweepLocked).sweep(0x351210, 0x0)
	/workdir/go/src/runtime/mgcsweep.go:607 +0x5f fp=0x3511f0 sp=0x351108 pc=0x118a005f
runtime.(*mcentral).uncacheSpan(0x33b4f0, 0x36b968)
...
runtime.gcMarkTermination()
	/workdir/go/src/runtime/mgc.go:1058 +0x6c fp=0x432f00 sp=0x432cf8 pc=0x111b006c
runtime.gcMarkDone()
	/workdir/go/src/runtime/mgc.go:918 +0x45 fp=0x432f58 sp=0x432f00 pc=0x11150045
runtime.gcBgMarkWorker()
	/workdir/go/src/runtime/mgc.go:1366 +0x66 fp=0x432fe0 sp=0x432f58 pc=0x111f0066
runtime.goexit()
	/workdir/go/src/runtime/asm_wasm.s:401 +0x1 fp=0x432fe8 sp=0x432fe0 pc=0x14350001
created by runtime.gcBgMarkStartWorkers
	/workdir/go/src/runtime/mgc.go:1158 +0x3
2022-08-09 16:09 js-wasm go@e1a8e0e0 go/internal/gcimporter (log)
runtime: marked free object in span 0x368228, elemsize=24 freeindex=0 (bad use of unsafe.Pointer? try -d=checkptr)
0x416000 alloc marked  
0x416018 alloc marked  
0x416030 alloc marked  
0x416048 alloc marked  
0x416060 alloc marked  
0x416078 alloc marked  
0x416090 alloc marked  
0x4160a8 alloc marked  
0x4160c0 alloc marked  
...
fatal error: found pointer to free object

goroutine 3 [running]:
runtime.throw({0x7d898, 0x1c})
	/workdir/go/src/runtime/panic.go:1047 +0x3 fp=0x4315e0 sp=0x4315b8 pc=0x12460003
runtime.(*mspan).reportZombies(0x368228)
	/workdir/go/src/runtime/mgcsweep.go:788 +0x60 fp=0x431660 sp=0x4315e0 pc=0x118b0060
runtime.(*sweepLocked).sweep(0x431780, 0x0)
	/workdir/go/src/runtime/mgcsweep.go:612 +0xfd fp=0x431748 sp=0x431660 pc=0x118a00fd
runtime.sweepone()
	/workdir/go/src/runtime/mgcsweep.go:370 +0x18 fp=0x4317a8 sp=0x431748 pc=0x11880018
runtime.bgsweep(0x438000)
	/workdir/go/src/runtime/mgcsweep.go:282 +0x2c fp=0x4317d0 sp=0x4317a8 pc=0x1185002c
runtime.gcenable.func1()
	/workdir/go/src/runtime/mgc.go:178 +0x2 fp=0x4317e0 sp=0x4317d0 pc=0x110a0002
runtime.goexit()
	/workdir/go/src/runtime/asm_wasm.s:401 +0x1 fp=0x4317e8 sp=0x4317e0 pc=0x14350001
created by runtime.gcenable
	/workdir/go/src/runtime/mgc.go:178 +0x8
2022-08-17 03:24 linux-amd64-staticlockranking go@c6be7103 cmd/trace (log)
207239  ======
0 : sysmon 1 0xd69998
1 : traceBuf 23 0xd80f98
2 : traceStackTab 42 0xd70ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe1ba?, 0x45eb45?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7ffe58068ed8 sp=0x7ffe58068ea8 pc=0x43a77d
...
cmd/trace.TestFoo(0xc000241520)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000052770 sp=0xc0000525b0 pc=0x751769
testing.tRunner(0xc000241520, 0xad0728)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000527c0 sp=0xc000052770 pc=0x51832b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000527e0 sp=0xc0000527c0 pc=0x5191ca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000527e8 sp=0xc0000527e0 pc=0x46f401
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-17 04:02 linux-amd64-staticlockranking go@c411886c cmd/trace (log)
206456  ======
0 : sysmon 1 0xd69998
1 : traceBuf 23 0xd80f98
2 : traceStackTab 42 0xd70ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe1ba?, 0x45eb45?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fccea0d1c20 sp=0x7fccea0d1bf0 pc=0x43a77d
...
cmd/trace.TestFoo(0xc0002311e0)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000052770 sp=0xc0000525b0 pc=0x751769
testing.tRunner(0xc0002311e0, 0xad0728)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000527c0 sp=0xc000052770 pc=0x51832b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000527e0 sp=0xc0000527c0 pc=0x5191ca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000527e8 sp=0xc0000527e0 pc=0x46f401
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-17 16:26 linux-amd64-staticlockranking go@5b0ce94c cmd/trace (log)
205725  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe2ba?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f3aee48cc20 sp=0x7f3aee48cbf0 pc=0x43a7bd
...
cmd/trace.TestFoo(0xc000437520)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000051f70 sp=0xc000051db0 pc=0x751609
testing.tRunner(0xc000437520, 0xad0828)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc000051fc0 sp=0xc000051f70 pc=0x5181cb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc000051fe0 sp=0xc000051fc0 pc=0x51906a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000051fe8 sp=0xc000051fe0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-17 16:48 linux-amd64-staticlockranking go@b11b4b4d cmd/trace (log)
206911  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45ebe5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fcd758b1c20 sp=0x7fcd758b1bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc000231a00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000053770 sp=0xc0000535b0 pc=0x751649
testing.tRunner(0xc000231a00, 0xad0788)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000537c0 sp=0xc000053770 pc=0x51820b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000537e0 sp=0xc0000537c0 pc=0x5190aa
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000537e8 sp=0xc0000537e0 pc=0x46f4a1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-17 17:40 linux-amd64-staticlockranking go@dea67a9b cmd/trace (log)
206798  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45ebe5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f9ff3d69c20 sp=0x7f9ff3d69bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc00021dd40)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00023f770 sp=0xc00023f5b0 pc=0x751669
testing.tRunner(0xc00021dd40, 0xad0788)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc00023f7c0 sp=0xc00023f770 pc=0x51820b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc00023f7e0 sp=0xc00023f7c0 pc=0x5190aa
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00023f7e8 sp=0xc00023f7e0 pc=0x46f4a1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-17 18:47 linux-amd64-staticlockranking go@57d05512 runtime/trace (log)
94330  ======
0 : traceStackTab 42 0x6fa658
1 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x5bd466?, 0x45d565?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fddef7fdc20 sp=0x7fddef7fdbf0 pc=0x4398dd
runtime.checkRanks(0xc0004036c0, 0x642358?, 0xffffffff000318ff?)
	/workdir/go/src/runtime/lockrank_on.go:156 +0x21a fp=0x7fddef7fdc80 sp=0x7fddef7fdc20 pc=0x40d93a
...
	/workdir/go/src/runtime/trace/trace_test.go:374 +0x125 fp=0xc0000537e0 sp=0xc0000536c8 pc=0x569de5
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000537e8 sp=0xc0000537e0 pc=0x46d581
created by runtime/trace_test.TestTraceStressStartStop
	/workdir/go/src/runtime/trace/trace_test.go:360 +0x12b

goroutine 19 [running]:
	goroutine running on other thread; stack unavailable
created by runtime.gcenable
	/workdir/go/src/runtime/mgc.go:179 +0xaa
2022-08-17 20:19 freebsd-amd64-12_3 go@9c2b481b compress/flate (log)
runtime: pointer 0xc0014a6000 to unallocated span span.base()=0xc0014a6000 span.limit=0xc0014c6020 span.state=0
runtime: found in object at *(0xc0012d6000+0x18058)
object=0xc0012d6000 s.base()=0xc001276000 s.limit=0xc001316508 s.spanclass=0 s.elemsize=663552 s.state=mSpanInUse
 *(object+0) = 0x0
 *(object+8) = 0x0
 *(object+16) = 0x0
 *(object+24) = 0x0
 *(object+32) = 0x0
 *(object+40) = 0x0
 *(object+48) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x559f67?, 0x6?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0xc000055df8 sp=0xc000055dc8 pc=0x4347bd
runtime.badPointer(0x827ccaf60, 0xc000055e70?, 0xc0012d6000, 0xc000055f00?)
	/tmp/workdir/go/src/runtime/mbitmap.go:316 +0x150 fp=0xc000055e48 sp=0xc000055df8 pc=0x413390
runtime.findObject(0xc00095af00?, 0xc00002cf00?, 0xc00004a800?)
	/tmp/workdir/go/src/runtime/mbitmap.go:359 +0xa6 fp=0xc000055e80 sp=0xc000055e48 pc=0x413526
runtime.scanobject(0xc00002bc40?, 0xc00002bc40)
	/tmp/workdir/go/src/runtime/mgcmark.go:1335 +0x165 fp=0xc000055f10 sp=0xc000055e80 pc=0x41dd45
runtime.gcDrain(0xc00002bc40, 0x3)
	/tmp/workdir/go/src/runtime/mgcmark.go:1105 +0x1d4 fp=0xc000055f70 sp=0xc000055f10 pc=0x41d5b4
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1307 +0xa5 fp=0xc000055fc0 sp=0xc000055f70 pc=0x419ca5
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0xc000055fc8 sp=0xc000055fc0 pc=0x464009
2022-08-17 23:18 linux-amd64-staticlockranking go@9709d92b cmd/trace (log)
208372  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45ebe5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7ffee1aee558 sp=0x7ffee1aee528 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc00021f380)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000053770 sp=0xc0000535b0 pc=0x751689
testing.tRunner(0xc00021f380, 0xad0788)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000537c0 sp=0xc000053770 pc=0x51822b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000537e0 sp=0xc0000537c0 pc=0x5190ca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000537e8 sp=0xc0000537e0 pc=0x46f4a1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 00:30 linux-amd64-staticlockranking go@b30ba3df cmd/trace (log)
207630  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45ebe5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7ff99abebc20 sp=0x7ff99abebbf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc000233ba0)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000235f70 sp=0xc000235db0 pc=0x751689
testing.tRunner(0xc000233ba0, 0xad0788)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc000235fc0 sp=0xc000235f70 pc=0x51822b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc000235fe0 sp=0xc000235fc0 pc=0x5190ca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000235fe8 sp=0xc000235fe0 pc=0x46f4a1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 02:51 linux-amd64-staticlockranking go@04d3d6bf cmd/trace (log)
208093  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f83c6597c20 sp=0x7f83c6597bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc000239520)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00043af70 sp=0xc00043adb0 pc=0x751649
testing.tRunner(0xc000239520, 0xad0788)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc00043afc0 sp=0xc00043af70 pc=0x5181eb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc00043afe0 sp=0xc00043afc0 pc=0x51908a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00043afe8 sp=0xc00043afe0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 05:32 linux-amd64-staticlockranking go@a2c2f06c cmd/trace (log)
208527  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fa4fd396c20 sp=0x7fa4fd396bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc0002236c0)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc0001cef70 sp=0xc0001cedb0 pc=0x751509
testing.tRunner(0xc0002236c0, 0xad0780)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0001cefc0 sp=0xc0001cef70 pc=0x5180ab
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0001cefe0 sp=0xc0001cefc0 pc=0x518f4a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001cefe8 sp=0xc0001cefe0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 14:53 linux-amd64-staticlockranking go@741ab7e8 cmd/trace (log)
208246  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe21a?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fe77e206c20 sp=0x7fe77e206bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc0000b8680)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00022bf70 sp=0xc00022bdb0 pc=0x751509
testing.tRunner(0xc0000b8680, 0xad0780)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc00022bfc0 sp=0xc00022bf70 pc=0x5180ab
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc00022bfe0 sp=0xc00022bfc0 pc=0x518f4a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00022bfe8 sp=0xc00022bfe0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 16:12 linux-amd64-staticlockranking go@b8b9e83e cmd/trace (log)
207507  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe20f?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fd2627fbc20 sp=0x7fd2627fbbf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc0002a31e0)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000057770 sp=0xc0000575b0 pc=0x751229
testing.tRunner(0xc0002a31e0, 0xad0760)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000577c0 sp=0xc000057770 pc=0x517dcb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000577e0 sp=0xc0000577c0 pc=0x518c6a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000577e8 sp=0xc0000577e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 16:15 linux-amd64-staticlockranking go@c7c78f1a cmd/trace (log)
207221  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe20f?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fa458a76c20 sp=0x7fa458a76bf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc00022b520)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc0000bd770 sp=0xc0000bd5b0 pc=0x751229
testing.tRunner(0xc00022b520, 0xad0760)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000bd7c0 sp=0xc0000bd770 pc=0x517dcb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000bd7e0 sp=0xc0000bd7c0 pc=0x518c6a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000bd7e8 sp=0xc0000bd7e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 16:35 linux-amd64-staticlockranking go@b2c0417c cmd/trace (log)
207708  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe20f?, 0x45eba5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7ff11fffec20 sp=0x7ff11fffebf0 pc=0x43a7fd
...
cmd/trace.TestFoo(0xc000102340)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000053770 sp=0xc0000535b0 pc=0x7511e9
testing.tRunner(0xc000102340, 0xad0760)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000537c0 sp=0xc000053770 pc=0x517dcb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000537e0 sp=0xc0000537c0 pc=0x518c6a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000537e8 sp=0xc0000537e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 17:09 linux-amd64-staticlockranking go@c82bbc0e cmd/trace (log)
193599  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe38f?, 0x45ebc5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f0155ffac20 sp=0x7f0155ffabf0 pc=0x43a81d
...
cmd/trace.TestFoo(0xc000544680)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc0000c3770 sp=0xc0000c35b0 pc=0x751209
testing.tRunner(0xc000544680, 0xad08e0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000c37c0 sp=0xc0000c3770 pc=0x517deb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000c37e0 sp=0xc0000c37c0 pc=0x518c8a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000c37e8 sp=0xc0000c37e0 pc=0x46f481
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 17:26 linux-amd64-staticlockranking go@b23d469e cmd/trace (log)
208048  ======
0 : sysmon 1 0xd6a998
1 : traceBuf 23 0xd81f98
2 : traceStackTab 42 0xd71ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fe38f?, 0x45ebc5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fcfb1580c20 sp=0x7fcfb1580bf0 pc=0x43a81d
...
cmd/trace.TestFoo(0xc000541a00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000053770 sp=0xc0000535b0 pc=0x751209
testing.tRunner(0xc000541a00, 0xad08e0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000537c0 sp=0xc000053770 pc=0x517deb
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000537e0 sp=0xc0000537c0 pc=0x518c8a
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000537e8 sp=0xc0000537e0 pc=0x46f481
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 18:18 linux-amd64-staticlockranking go@1aa5730f cmd/trace (log)
207493  ======
0 : sysmon 1 0xd66998
1 : traceBuf 23 0xd7df98
2 : traceStackTab 42 0xd6def8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fb38f?, 0x45ec45?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f94ab536c20 sp=0x7f94ab536bf0 pc=0x43a87d
...
cmd/trace.TestFoo(0xc000103a00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000052770 sp=0xc0000525b0 pc=0x74e029
testing.tRunner(0xc000103a00, 0xacd8d0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000527c0 sp=0xc000052770 pc=0x51776b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000527e0 sp=0xc0000527c0 pc=0x5185ea
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000527e8 sp=0xc0000527e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 21:22 linux-amd64-staticlockranking go@5fcddf39 cmd/trace (log)
208328  ======
0 : sysmon 1 0xd66998
1 : traceBuf 23 0xd7df98
2 : traceStackTab 42 0xd6def8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fb38f?, 0x45ec45?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f361c11ec20 sp=0x7f361c11ebf0 pc=0x43a87d
...
cmd/trace.TestFoo(0xc000225860)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00032f770 sp=0xc00032f5b0 pc=0x74e029
testing.tRunner(0xc000225860, 0xacd8d0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc00032f7c0 sp=0xc00032f770 pc=0x51776b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc00032f7e0 sp=0xc00032f7c0 pc=0x5185ea
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00032f7e8 sp=0xc00032f7e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-18 21:54 linux-amd64-staticlockranking go@9ad77203 cmd/trace (log)
207715  ======
0 : sysmon 1 0xd66998
1 : traceBuf 23 0xd7df98
2 : traceStackTab 42 0xd6def8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7fb38f?, 0x45ec45?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f1039d76c20 sp=0x7f1039d76bf0 pc=0x43a87d
...
cmd/trace.TestFoo(0xc000103a00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00023f770 sp=0xc00023f5b0 pc=0x74e029
testing.tRunner(0xc000103a00, 0xacd8d0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc00023f7c0 sp=0xc00023f770 pc=0x51776b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc00023f7e0 sp=0xc00023f7c0 pc=0x5185ea
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00023f7e8 sp=0xc00023f7e0 pc=0x46f461
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-19 01:34 linux-amd64-staticlockranking go@ee833ed7 cmd/trace (log)
210886  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8f0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f4cb4169c20 sp=0x7f4cb4169bf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc000220b60)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000156770 sp=0xc0001565b0 pc=0x752e49
testing.tRunner(0xc000220b60, 0xad1df0)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0001567c0 sp=0xc000156770 pc=0x519b4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0001567e0 sp=0xc0001567c0 pc=0x51a9ca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001567e8 sp=0xc0001567e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-19 21:29 linux-amd64-staticlockranking go@375a80c9 cmd/trace (log)
211255  ======
0 : sysmon 1 0xd6c978
1 : traceBuf 23 0xd83f78
2 : traceStackTab 42 0xd73ed8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fe4ec2b8c20 sp=0x7fe4ec2b8bf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc000225860)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000052770 sp=0xc0000525b0 pc=0x752d89
testing.tRunner(0xc000225860, 0xad1d10)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000527c0 sp=0xc000052770 pc=0x519c4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000527e0 sp=0xc0000527c0 pc=0x51aaca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000527e8 sp=0xc0000527e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-22 11:46 linux-amd64-staticlockranking go@9a9aad44 cmd/trace (log)
211003  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7ff11fffec20 sp=0x7ff11fffebf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc0000db040)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000051770 sp=0xc0000515b0 pc=0x752d09
testing.tRunner(0xc0000db040, 0xad1d08)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0000517c0 sp=0xc000051770 pc=0x519c4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0000517e0 sp=0xc0000517c0 pc=0x51aaca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000517e8 sp=0xc0000517e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-22 14:48 linux-amd64-staticlockranking go@6bdca820 cmd/trace (log)
210352  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fd4fdf85c20 sp=0x7fd4fdf85bf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc0002b3860)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000225f70 sp=0xc000225db0 pc=0x752d09
testing.tRunner(0xc0002b3860, 0xad1d08)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc000225fc0 sp=0xc000225f70 pc=0x519c4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc000225fe0 sp=0xc000225fc0 pc=0x51aaca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000225fe8 sp=0xc000225fe0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-22 16:48 linux-amd64-staticlockranking go@a10da772 cmd/trace (log)
211392  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fe6d67fbc20 sp=0x7fe6d67fbbf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc0002a1a00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000247770 sp=0xc0002475b0 pc=0x752ca9
testing.tRunner(0xc0002a1a00, 0xad1d08)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0002477c0 sp=0xc000247770 pc=0x519c4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0002477e0 sp=0xc0002477c0 pc=0x51aaca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0002477e8 sp=0xc0002477e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-22 18:50 linux-amd64-staticlockranking go@e21c1f78 cmd/trace (log)
211500  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f100b2bec20 sp=0x7f100b2bebf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc000543380)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc0001cf770 sp=0xc0001cf5b0 pc=0x752ca9
testing.tRunner(0xc000543380, 0xad1d08)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0001cf7c0 sp=0xc0001cf770 pc=0x519c4b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0001cf7e0 sp=0xc0001cf7c0 pc=0x51aaca
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001cf7e8 sp=0xc0001cf7e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-23 03:09 linux-amd64-staticlockranking go@0a52d806 cmd/trace (log)
211043  ======
0 : sysmon 1 0xd6b998
1 : traceBuf 23 0xd82f98
2 : traceStackTab 42 0xd72ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f091fffec20 sp=0x7f091fffebf0 pc=0x439e3d
...
cmd/trace.TestFoo(0xc00020ed00)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000263770 sp=0xc0002635b0 pc=0x752a49
testing.tRunner(0xc00020ed00, 0xad1d08)
	/workdir/go/src/testing/testing.go:1446 +0x10b fp=0xc0002637c0 sp=0xc000263770 pc=0x519c2b
testing.(*T).Run.func1()
	/workdir/go/src/testing/testing.go:1493 +0x2a fp=0xc0002637e0 sp=0xc0002637c0 pc=0x51aaaa
runtime.goexit()
	/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0002637e8 sp=0xc0002637e0 pc=0x46f4c1
created by testing.(*T).Run
	/workdir/go/src/testing/testing.go:1493 +0x35f
2022-08-26 17:48 darwin-amd64-10_14 go@296c40db compress/flate (log)
runtime: pointer 0xc000f68000 to unallocated span span.base()=0xc000f68000 span.limit=0xc000f77fff span.state=0
runtime: found in object at *(0xc0013c0000+0x0)
object=0xc0013c0000 s.base()=0xc0013a0000 s.limit=0xc001440508 s.spanclass=0 s.elemsize=663552 s.state=mSpanInUse
 *(object+0) = 0x0 <==
 *(object+8) = 0x0
 *(object+16) = 0x0
 *(object+24) = 0x0
 *(object+32) = 0x0
 *(object+40) = 0x0
 *(object+48) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x115b36f?, 0x6?})
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/panic.go:1047 +0x5d fp=0x7000090edce8 sp=0x7000090edcb8 pc=0x1033e7d
runtime.badPointer(0x1409a20, 0x7000090edd60?, 0xc0013c0000, 0x1024328?)
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mbitmap.go:314 +0x150 fp=0x7000090edd38 sp=0x7000090edce8 pc=0x1013210
runtime.findObject(0xc0030c4000?, 0x102fcff?, 0xc000031738?)
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mbitmap.go:357 +0xa6 fp=0x7000090edd70 sp=0x7000090edd38 pc=0x10133a6
runtime.scanobject(0xc000031738?, 0xc000031738)
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mgcmark.go:1337 +0x165 fp=0x7000090ede00 sp=0x7000090edd70 pc=0x101d8c5
runtime.gcDrainN(0xc000031738, 0x10000)
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mgcmark.go:1194 +0x192 fp=0x7000090ede30 sp=0x7000090ede00 pc=0x101d512
runtime.gcAssistAlloc1(0xc000117860, 0xc000331910?)
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mgcmark.go:569 +0x12e fp=0x7000090ede98 sp=0x7000090ede30 pc=0x101bfae
runtime.gcAssistAlloc.func1()
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/mgcmark.go:476 +0x25 fp=0x7000090edeb8 sp=0x7000090ede98 pc=0x101be65
runtime.systemstack()
	/var/folders/9w/4l2_g3kx01x199n37fbmv3s80000gn/T/workdir-host-darwin-amd64-10_14/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x7000090edec0 sp=0x7000090edeb8 pc=0x1065309
2022-09-02 19:09 windows-arm64-10 go@55ca6a20 runtime/trace (log)
fatal error: unexpected signal during runtime execution
[signal 0xc0000005 code=0x0 addr=0x0 pc=0x7ff6f49eb8e4]

runtime stack:
runtime.throw({0x7ff6f4b4c65b?, 0x0?})
	C:/workdir/go/src/runtime/panic.go:1047 +0x40 fp=0xea793ff1b0 sp=0xea793ff180 pc=0x7ff6f49c73a0
runtime.sigpanic()
	C:/workdir/go/src/runtime/signal_windows.go:251 +0x220 fp=0xea793ff200 sp=0xea793ff1b0 pc=0x7ff6f49dcc90
runtime.gentraceback(0x4000041520?, 0x4000024000?, 0xea793ff5a8?, 0x40003c4000, 0x0, 0xea793ff5a8, 0x40, 0x0, 0x0?, 0x6)
	C:/workdir/go/src/runtime/traceback.go:162 +0x304 fp=0xea793ff540 sp=0xea793ff210 pc=0x7ff6f49eb8e4
runtime.sigprof(0x7ff6f49f71d8, 0x20c?, 0xea793ff7f0?, 0x40003c4000, 0x40003c0000)
	C:/workdir/go/src/runtime/proc.go:4626 +0xf8 fp=0xea793ff7b0 sp=0xea793ff540 pc=0x7ff6f49d3618
runtime.profilem(0x40003c0000, 0x20c?)
	C:/workdir/go/src/runtime/os_windows.go:1228 +0x110 fp=0xea793ffb90 sp=0xea793ff7b0 pc=0x7ff6f49c43d0
runtime.profileLoop()
	C:/workdir/go/src/runtime/os_windows.go:1284 +0x178 fp=0xea793ffc10 sp=0xea793ffb90 pc=0x7ff6f49c4578
runtime.mstart1()
	C:/workdir/go/src/runtime/proc.go:1425 +0x88 fp=0xea793ffc40 sp=0xea793ffc10 pc=0x7ff6f49cc308
runtime.mstart0()
	C:/workdir/go/src/runtime/proc.go:1382 +0x68 fp=0xea793ffc70 sp=0xea793ffc40 pc=0x7ff6f49cc268
runtime.mstart()
	C:/workdir/go/src/runtime/asm_arm64.s:128 +0x10 fp=0xea793ffc80 sp=0xea793ffc70 pc=0x7ff6f49f70a0
2022-09-05 08:08 windows-amd64-longtest go@af7f4176 runtime/pprof (log)
fatal error: unexpected signal during runtime execution
[signal 0xc0000005 code=0x0 addr=0x0 pc=0x3ac66f]

runtime stack:
runtime.throw({0x51017b?, 0x5e6150?})
	C:/workdir/go/src/runtime/panic.go:1047 +0x65 fp=0x69bd3fed18 sp=0x69bd3fece8 pc=0x388645
runtime.sigpanic()
	C:/workdir/go/src/runtime/signal_windows.go:251 +0x213 fp=0x69bd3fed60 sp=0x69bd3fed18 pc=0x39d453
runtime.gentraceback(0xffffffdccc0fa9c0?, 0x69bd3ff130?, 0x8?, 0xc000052820, 0x0, 0x69bd3ff130, 0x40, 0x0, 0x0?, 0x6)
	C:/workdir/go/src/runtime/traceback.go:162 +0x4cf fp=0x69bd3ff0d0 sp=0x69bd3fed60 pc=0x3ac66f
runtime.sigprof(0x3918bf, 0x198?, 0x69bd3ff370?, 0xc000052820, 0xc00005c800)
	C:/workdir/go/src/runtime/proc.go:4626 +0x11d fp=0x69bd3ff340 sp=0x69bd3ff0d0 pc=0x3944bd
runtime.profilem(0xc00005c800, 0x198?)
	C:/workdir/go/src/runtime/os_windows.go:1228 +0xff fp=0x69bd3ff860 sp=0x69bd3ff340 pc=0x38553f
runtime.profileLoop()
	C:/workdir/go/src/runtime/os_windows.go:1284 +0x1d2 fp=0x69bd3ff8d8 sp=0x69bd3ff860 pc=0x385752
runtime.mstart1()
	C:/workdir/go/src/runtime/proc.go:1425 +0x9d fp=0x69bd3ff900 sp=0x69bd3ff8d8 pc=0x38d7fd
runtime.mstart0()
	C:/workdir/go/src/runtime/proc.go:1382 +0x6c fp=0x69bd3ff928 sp=0x69bd3ff900 pc=0x38d72c
runtime.mstart()
	C:/workdir/go/src/runtime/asm_amd64.s:390 +0x5 fp=0x69bd3ff930 sp=0x69bd3ff928 pc=0x3b8305
2022-09-07 06:18 freebsd-amd64-13_0 go@6375f508 compress/flate (log)
runtime: pointer 0xc00191a000 to unused region of span span.base()=0xc000968000 span.limit=0xc000977fff span.state=1
runtime: found in object at *(0xc001828000+0x1c078)
object=0xc001828000 s.base()=0xc0017e8000 s.limit=0xc001888508 s.spanclass=0 s.elemsize=663552 s.state=mSpanInUse
 *(object+0) = 0x0
 *(object+8) = 0x0
 *(object+16) = 0x0
 *(object+24) = 0x0
 *(object+32) = 0x0
 *(object+40) = 0x0
 *(object+48) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x55a430?, 0x6?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0xc00005bdf8 sp=0xc00005bdc8 pc=0x433ebd
runtime.badPointer(0x827bb1898, 0xc00005be70?, 0xc001828000, 0x4242c8?)
	/tmp/workdir/go/src/runtime/mbitmap.go:314 +0x150 fp=0xc00005be48 sp=0xc00005bdf8 pc=0x413170
runtime.findObject(0xc0000141e0?, 0x0?, 0x0?)
	/tmp/workdir/go/src/runtime/mbitmap.go:357 +0xa6 fp=0xc00005be80 sp=0xc00005be48 pc=0x413306
runtime.scanobject(0xc000027238?, 0xc000027238)
	/tmp/workdir/go/src/runtime/mgcmark.go:1337 +0x165 fp=0xc00005bf10 sp=0xc00005be80 pc=0x41d865
runtime.gcDrain(0xc000027238, 0x3)
	/tmp/workdir/go/src/runtime/mgcmark.go:1105 +0x1d4 fp=0xc00005bf70 sp=0xc00005bf10 pc=0x41d0d4
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1307 +0xa5 fp=0xc00005bfc0 sp=0xc00005bf70 pc=0x4197c5
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0xc00005bfc8 sp=0xc00005bfc0 pc=0x4642e9
2022-09-07 16:39 aix-ppc64 go@027ff3f4 (log)
runtime: pointer 0xa00010000664e00 to unallocated span span.base()=0xa00010000664000 span.limit=0xa00010000667f00 span.state=0
runtime: found in object at *(0xa000100000c4600+0x0)
object=0xa000100000c4600 s.base()=0xa000100000c4000 s.limit=0xa000100000c5e00 s.spanclass=72 s.elemsize=1536 s.state=mSpanInUse
 *(object+0) = 0x0 <==
 *(object+8) = 0x0
 *(object+16) = 0x0
 *(object+24) = 0x0
 *(object+32) = 0x0
 *(object+40) = 0x0
 *(object+48) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x100160e36, 0x3e})
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/panic.go:1198 +0x68 fp=0x11027d508 sp=0x11027d4c8 pc=0x10003a0b8
runtime.badPointer(0xa00000026e74b00, 0xa00010000664e00, 0xa000100000c4600, 0x0)
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/mbitmap.go:367 +0x174 fp=0x11027d560 sp=0x11027d508 pc=0x100014514
runtime.findObject(0xa00010000664e00, 0xa000100000c4600, 0x0)
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/mbitmap.go:409 +0x104 fp=0x11027d5a8 sp=0x11027d560 pc=0x1000146d4
runtime.scanobject(0xa000100000c4600, 0xa0001000003be98)
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/mgcmark.go:1280 +0x1ec fp=0x11027d648 sp=0x11027d5a8 pc=0x100020b8c
runtime.gcDrain(0xa0001000003be98, 0x3)
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/mgcmark.go:1047 +0x25c fp=0x11027d6b0 sp=0x11027d648 pc=0x10002031c
runtime.gcBgMarkWorker.func2()
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/mgc.go:1269 +0xbc fp=0x11027d720 sp=0x11027d6b0 pc=0x10001ca2c
runtime.systemstack()
	/ramdisk8GB/workdir-host-aix-ppc64-osuosl/go1.4/src/runtime/asm_ppc64x.s:250 +0x68 fp=0x11027d740 sp=0x11027d720 pc=0x10006c0e8
2022-09-16 16:32 linux-386-buster go@686b38b5 compress/flate (log)
runtime: pointer 0x9ec6000 to unallocated span span.base()=0x9ec6000 span.limit=0x9ed5fff span.state=0
runtime: found in object at *(0xa7a2000+0x3c)
object=0xa7a2000 s.base()=0xa702000 s.limit=0xa7a2484 s.spanclass=0 s.elemsize=663552 s.state=mSpanInUse
 *(object+0) = 0x0
 *(object+4) = 0x0
 *(object+8) = 0x0
 *(object+12) = 0x0
 *(object+16) = 0x0
 *(object+20) = 0x0
 *(object+24) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x8192faf, 0x3e})
	/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x983bec4 sp=0x983beb0 pc=0x807ed6d
runtime.badPointer(0xef819a50, 0x9ec6000, 0xa7a2000, 0x3c)
	/workdir/go/src/runtime/mbitmap.go:314 +0xa2 fp=0x983bee4 sp=0x983bec4 pc=0x8058582
runtime.findObject(0x9ec6000, 0xa7a2000, 0x3c)
	/workdir/go/src/runtime/mbitmap.go:357 +0x9f fp=0x983befc sp=0x983bee4 pc=0x80587df
runtime.scanobject(0xa7a2000, 0x942c95c)
	/workdir/go/src/runtime/mgcmark.go:1335 +0x16e fp=0x983bf4c sp=0x983befc pc=0x8064c8e
runtime.gcDrain(0x942c95c, 0x3)
	/workdir/go/src/runtime/mgcmark.go:1103 +0x24f fp=0x983bfb0 sp=0x983bf4c pc=0x80643ef
runtime.gcBgMarkWorker.func2()
	/workdir/go/src/runtime/mgc.go:1339 +0xaf fp=0x983bfdc sp=0x983bfb0 pc=0x805fcef
runtime.systemstack()
	/workdir/go/src/runtime/asm_386.s:370 +0x41 fp=0x983bfe0 sp=0x983bfdc pc=0x80af361

watchflakes

@rsc rsc changed the title runtime: flaky corruption in tests runtime: corruption in tests Sep 20, 2022
@gopherbot
Copy link

gopherbot commented Sep 21, 2022

Found new dashboard test flakes for:

#!watchflakes
default <- `runtime.throw\(` || `fatal error:`
2022-08-22 22:29 linux-amd64-staticlockranking go@ae303ff2 cmd/trace.TestFoo (log)
212720  ======
0 : sysmon 1 0xd6c998
1 : traceBuf 23 0xd83f98
2 : traceStackTab 42 0xd73ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7fefe3385c20 sp=0x7fefe3385bf0 pc=0x439e3d
...
	/workdir/go/src/runtime/trace.go:879 +0xc5 fp=0xc000051450 sp=0xc000051408 pc=0x45d825
runtime.StartTrace()
	/workdir/go/src/runtime/trace.go:242 +0xa9 fp=0xc0000514d0 sp=0xc000051450 pc=0x45bc09
runtime/trace.Start({0xb3adc0?, 0xc0000a7560})
	/workdir/go/src/runtime/trace/trace.go:125 +0x97 fp=0xc000051528 sp=0xc0000514d0 pc=0x50bf17
cmd/trace.traceProgram(0xc0002a9380, 0xad1d00, {0x7f636b, 0x7})
	/workdir/go/src/cmd/trace/annotations_test.go:334 +0x5f fp=0xc0000515b0 sp=0xc000051528 pc=0x75031f
cmd/trace.TestFoo(0xc0002a9380)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc000051770 sp=0xc0000515b0 pc=0x752a49
testing.tRunner(0xc0002a9380, 0xad1d08)
2022-08-22 23:16 linux-amd64-staticlockranking go@cd528184 cmd/trace.TestFoo (log)
212262  ======
0 : sysmon 1 0xd6b998
1 : traceBuf 23 0xd82f98
2 : traceStackTab 42 0xd72ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f91427fbc20 sp=0x7f91427fbbf0 pc=0x439e3d
...
	/workdir/go/src/runtime/trace.go:879 +0xc5 fp=0xc0000d1c50 sp=0xc0000d1c08 pc=0x45d825
runtime.StartTrace()
	/workdir/go/src/runtime/trace.go:242 +0xa9 fp=0xc0000d1cd0 sp=0xc0000d1c50 pc=0x45bc09
runtime/trace.Start({0xb3ada0?, 0xc0002a6660})
	/workdir/go/src/runtime/trace/trace.go:125 +0x97 fp=0xc0000d1d28 sp=0xc0000d1cd0 pc=0x50bf17
cmd/trace.traceProgram(0xc00022a820, 0xad1d00, {0x7f636b, 0x7})
	/workdir/go/src/cmd/trace/annotations_test.go:334 +0x5f fp=0xc0000d1db0 sp=0xc0000d1d28 pc=0x75031f
cmd/trace.TestFoo(0xc00022a820)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc0000d1f70 sp=0xc0000d1db0 pc=0x752a49
testing.tRunner(0xc00022a820, 0xad1d08)
2022-08-22 23:36 linux-amd64-staticlockranking go@e0388c92 cmd/trace.TestFoo (log)
210167  ======
0 : sysmon 1 0xd6b998
1 : traceBuf 23 0xd82f98
2 : traceStackTab 42 0xd72ef8
3 : gscan 34 0x0
fatal error: lock ordering problem

runtime stack:
runtime.throw({0x7ff8b0?, 0x45d7e5?})
	/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f93d994cc20 sp=0x7f93d994cbf0 pc=0x439e3d
...
	/workdir/go/src/runtime/trace.go:879 +0xc5 fp=0xc00023e450 sp=0xc00023e408 pc=0x45d825
runtime.StartTrace()
	/workdir/go/src/runtime/trace.go:242 +0xa9 fp=0xc00023e4d0 sp=0xc00023e450 pc=0x45bc09
runtime/trace.Start({0xb3ada0?, 0xc000205d70})
	/workdir/go/src/runtime/trace/trace.go:125 +0x97 fp=0xc00023e528 sp=0xc00023e4d0 pc=0x50bf17
cmd/trace.traceProgram(0xc0000d24e0, 0xad1d00, {0x7f636b, 0x7})
	/workdir/go/src/cmd/trace/annotations_test.go:334 +0x5f fp=0xc00023e5b0 sp=0xc00023e528 pc=0x75031f
cmd/trace.TestFoo(0xc0000d24e0)
	/workdir/go/src/cmd/trace/trace_test.go:190 +0x49 fp=0xc00023e770 sp=0xc00023e5b0 pc=0x752a49
testing.tRunner(0xc0000d24e0, 0xad1d08)
2022-08-26 15:44 linux-mips-rtrk net@83b083e8 go@bd5595d7 (log)
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x31c80]

runtime stack:
runtime.throw({0x1a26ae, 0x2a})
	/tmp/gobuilder-mips64/go1.4/src/runtime/panic.go:1198 +0x6c
runtime.sigpanic()
	/tmp/gobuilder-mips64/go1.4/src/runtime/signal_unix.go:719 +0x328
runtime.readgstatus(...)
	/tmp/gobuilder-mips64/go1.4/src/runtime/proc.go:889
runtime.markroot(0xc000028e98, 0x96)
	/tmp/gobuilder-mips64/go1.4/src/runtime/mgcmark.go:198 +0x228
runtime.gcDrain(0xc000028e98, 0x3)
	/tmp/gobuilder-mips64/go1.4/src/runtime/mgcmark.go:1013 +0x534
runtime.gcBgMarkWorker.func2()
	/tmp/gobuilder-mips64/go1.4/src/runtime/mgc.go:1269 +0xe4
runtime.systemstack()
	/tmp/gobuilder-mips64/go1.4/src/runtime/asm_mips64x.s:191 +0x7c

watchflakes

@mknyszek mknyszek added this to the Go1.20 milestone Sep 28, 2022
@rsc
Copy link
Contributor Author

rsc commented Sep 28, 2022

These are old failures. The goal for this bug is to be reopened each time there's a new otherwise uncategorized runtime.throw (instead of, say, creating a new bug pointing at a test in package strings or wherever it manifested), and then we make a more specific issue for it, and close this one. And then the next time there's an uncategorized runtime.throw, watchflakes posts and reopens. So I will close this one right now.

@rsc rsc closed this as completed Sep 28, 2022
@gopherbot
Copy link

gopherbot commented Oct 4, 2022

Found new dashboard test flakes for:

#!watchflakes
default <- `runtime.throw\(` || `fatal error:`
2022-09-29 19:35 linux-386-softfloat go@545adcfe compress/flate.TestDeflateInflate (log)
runtime: pointer 0x91e8000 to unallocated span span.base()=0x91e8000 span.limit=0x91f7fff span.state=0
runtime: found in object at *(0x9fb8000+0x3c)
object=0x9fb8000 s.base()=0x9f18000 s.limit=0x9fb8484 s.spanclass=0 s.elemsize=663552 s.state=mSpanInUse
 *(object+0) = 0x0
 *(object+4) = 0x0
 *(object+8) = 0x0
 *(object+12) = 0x0
 *(object+16) = 0x0
 *(object+20) = 0x0
 *(object+24) = 0x0
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x819834d, 0x3e})
	/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x8d89ec4 sp=0x8d89eb0 pc=0x807fd3d
runtime.badPointer(0xef54ea20, 0x91e8000, 0x9fb8000, 0x3c)
	/workdir/go/src/runtime/mbitmap.go:314 +0xa2 fp=0x8d89ee4 sp=0x8d89ec4 pc=0x80589a2
runtime.findObject(0x91e8000, 0x9fb8000, 0x3c)
	/workdir/go/src/runtime/mbitmap.go:357 +0x9f fp=0x8d89efc sp=0x8d89ee4 pc=0x8058bff
runtime.scanobject(0x9fb8000, 0x8828f5c)
...
	/workdir/go/src/compress/flate/deflate.go:666 +0x23 fp=0x89fad30 sp=0x89fad10 pc=0x8136703
compress/flate.testSync(0x8882e10, 0x6, {0x8269086, 0x1, 0x1}, {0x88b8078, 0x2})
	/workdir/go/src/compress/flate/deflate_test.go:265 +0x417 fp=0x89faea0 sp=0x89fad30 pc=0x813e897
compress/flate.testToFromWithLevelAndLimit(0x8882e10, 0x6, {0x8269086, 0x1, 0x1}, {0x88b8078, 0x2}, 0x0)
	/workdir/go/src/compress/flate/deflate_test.go:369 +0x4bc fp=0x89faf14 sp=0x89faea0 pc=0x813faec
compress/flate.testToFromWithLimit(0x8882e10, {0x8269086, 0x1, 0x1}, {0x88b8078, 0x2}, {0x0, 0x0, 0x0, 0x0, ...})
	/workdir/go/src/compress/flate/deflate_test.go:374 +0x65 fp=0x89faf3c sp=0x89faf14 pc=0x813fb65
compress/flate.TestDeflateInflate(0x8882e10)
	/workdir/go/src/compress/flate/deflate_test.go:386 +0x11d fp=0x89faf9c sp=0x89faf3c pc=0x813fd0d
testing.tRunner(0x8882e10, 0x819ab88)

watchflakes

@cherrymui
Copy link
Member

cherrymui commented Oct 4, 2022

This is #54596 . I updated pattern there.

For this issue, does watchflakes suppose to reopen it as it found a new failure? @rsc

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one.
Projects
Status: Done
Status: Done
Development

No branches or pull requests

4 participants