runtime: deadlocks due to missed `___lwp_park60` wakeups on NetBSD

[bcmills]

#!watchflakes
default <- goos == "netbsd" && (`SIGQUIT` || `panic: test timed out`)

In CL 442478 I fixed (or at least attempted to fix) Go's use of the wait6 system call on NetBSD for the (*os.Process).Wait hang tracked in #50138.

In the process of tracking that down, I observed a possible missed-wakeup bug or deadlock in a C-only reproducer program (#50138 (comment)).

I am now seeing deadlocks in runtime.gopark on NetBSD builders and SlowBots, and I was able to reproduce the failure mode by running go test os/exec -timeout=10s in a loop on the builder (go test os/exec normally completes within about half a second), and that symptom is at least superficially similar to the hang in the C program (both involve hangs in semaphore-based wakeup paths).

This is the goroutine dump I obtained (with GOTRACEBACK=crash):

panic: test timed out after 10s

goroutine 151 [running]:
panic({0x6f5040, 0xc00032c110})
        /tmp/workdir/go/src/runtime/panic.go:987 +0x3bb fp=0xc000269f98 sp=0xc000269ed8 pc=0x43621b
testing.(*M).startAlarm.func1()
        /tmp/workdir/go/src/testing/testing.go:2107 +0x8e fp=0xc000269fe0 sp=0xc000269f98 pc=0x4e8a0e
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000269fe8 sp=0xc000269fe0 pc=0x469c01
created by time.goFunc
        /tmp/workdir/go/src/time/sleep.go:176 +0x32

goroutine 1 [chan receive]:
runtime.gopark(0xc000074000?, 0x0?, 0x0?, 0x0?, 0xc0000061a0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00005b820 sp=0xc00005b800 pc=0x4392b6
runtime.chanrecv(0xc000094540, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00005b8b0 sp=0xc00005b820 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0xc0001861a8?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00005b8d8 sp=0xc00005b8b0 pc=0x407478
testing.tRunner.func1()
        /tmp/workdir/go/src/testing/testing.go:1467 +0x4a5 fp=0xc00005ba40 sp=0xc00005b8d8 pc=0x4e5165
testing.tRunner(0xc000134680, 0xc00005bb58)
        /tmp/workdir/go/src/testing/testing.go:1507 +0x144 fp=0xc00005ba90 sp=0xc00005ba40 pc=0x4e4c44
testing.runTests(0xc00011ea00?, {0x9a25a0, 0x21, 0x21}, {0x0?, 0xc0000061a0?, 0x9a8120?})
        /tmp/workdir/go/src/testing/testing.go:1911 +0x489 fp=0xc00005bb88 sp=0xc00005ba90 pc=0x4e7869
testing.(*M).Run(0xc00011ea00)
        /tmp/workdir/go/src/testing/testing.go:1783 +0x5f7 fp=0xc00005bd98 sp=0xc00005bb88 pc=0x4e62d7
os/exec_test.TestMain(0xffffffffffffffff?)
        /tmp/workdir/go/src/os/exec/exec_test.go:79 +0xfc fp=0xc00005bec8 sp=0xc00005bd98 pc=0x6c8c9c
main.main()
        _testmain.go:117 +0x1d3 fp=0xc00005bf80 sp=0xc00005bec8 pc=0x6d1993
runtime.main()
        /tmp/workdir/go/src/runtime/proc.go:250 +0x207 fp=0xc00005bfe0 sp=0xc00005bf80 pc=0x438ee7
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00005bfe8 sp=0xc00005bfe0 pc=0x469c01

goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000048fb0 sp=0xc000048f90 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.forcegchelper()
        /tmp/workdir/go/src/runtime/proc.go:305 +0xb0 fp=0xc000048fe0 sp=0xc000048fb0 pc=0x439150
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000048fe8 sp=0xc000048fe0 pc=0x469c01
created by runtime.init.5
        /tmp/workdir/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000049780 sp=0xc000049760 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.bgsweep(0x0?)
        /tmp/workdir/go/src/runtime/mgcsweep.go:320 +0xde fp=0xc0000497c8 sp=0xc000049780 pc=0x42489e
runtime.gcenable.func1()
        /tmp/workdir/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000497e0 sp=0xc0000497c8 pc=0x419ae6
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000497e8 sp=0xc0000497e0 pc=0x469c01
created by runtime.gcenable
        /tmp/workdir/go/src/runtime/mgc.go:178 +0x6b

goroutine 4 [GC scavenge wait]:
runtime.gopark(0x1980c7dd917?, 0x15c1154d?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000049f70 sp=0xc000049f50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.(*scavengerState).park(0x9a81a0)
        /tmp/workdir/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc000049fa0 sp=0xc000049f70 pc=0x4228b3
runtime.bgscavenge(0x0?)
        /tmp/workdir/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc000049fc8 sp=0xc000049fa0 pc=0x422ea5
runtime.gcenable.func2()
        /tmp/workdir/go/src/runtime/mgc.go:179 +0x26 fp=0xc000049fe0 sp=0xc000049fc8 pc=0x419a86
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000049fe8 sp=0xc000049fe0 pc=0x469c01
created by runtime.gcenable
        /tmp/workdir/go/src/runtime/mgc.go:179 +0xaa

goroutine 5 [finalizer wait]:
runtime.gopark(0x0?, 0x78dc40?, 0x0?, 0xc0?, 0x1000000010?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000048628 sp=0xc000048608 pc=0x4392b6
runtime.runfinq()
        /tmp/workdir/go/src/runtime/mfinal.go:193 +0x107 fp=0xc0000487e0 sp=0xc000048628 pc=0x418b87
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000487e8 sp=0xc0000487e0 pc=0x469c01
created by runtime.createfing
        /tmp/workdir/go/src/runtime/mfinal.go:163 +0x45

goroutine 30 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc000044618?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000044570 sp=0xc000044550 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000044600 sp=0xc000044570 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000044628 sp=0xc000044600 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000044650 sp=0xc000044628 pc=0x4e5bcb
testing.(*T).Parallel(0xc000083ba0)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc0000446b8 sp=0xc000044650 pc=0x4e4930
os/exec_test.TestCredentialNoSetGroups(0xc000083ba0)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:43 +0x33 fp=0xc000044770 sp=0xc0000446b8 pc=0x6c6d13
testing.tRunner(0xc000083ba0, 0x78dca0)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0000447c0 sp=0xc000044770 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0000447e0 sp=0xc0000447c0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000447e8 sp=0xc0000447e0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 31 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc000044e28?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000044d80 sp=0xc000044d60 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000044e10 sp=0xc000044d80 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000044e38 sp=0xc000044e10 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000044e60 sp=0xc000044e38 pc=0x4e5bcb
testing.(*T).Parallel(0xc000083d40)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000044ec8 sp=0xc000044e60 pc=0x4e4930
os/exec_test.TestWaitid(0xc000083d40)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:78 +0x33 fp=0xc000044f70 sp=0xc000044ec8 pc=0x6c7053
testing.tRunner(0xc000083d40, 0x78dd70)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000044fc0 sp=0xc000044f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000044fe0 sp=0xc000044fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000044fe8 sp=0xc000044fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 32 [chan receive]:
runtime.gopark(0xc0000ac9c0?, 0xc000082ea0?, 0xe0?, 0x2e?, 0xc000101200?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000056d50 sp=0xc000056d30 pc=0x4392b6
runtime.chanrecv(0xc000072d90, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000056de0 sp=0xc000056d50 pc=0x40793d
runtime.chanrecv1(0xc000092540?, 0x75a568?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000056e08 sp=0xc000056de0 pc=0x407478
testing.tRunner.func1()
        /tmp/workdir/go/src/testing/testing.go:1467 +0x4a5 fp=0xc000056f70 sp=0xc000056e08 pc=0x4e5165
testing.tRunner(0xc0000d4000, 0x78dd10)
        /tmp/workdir/go/src/testing/testing.go:1507 +0x144 fp=0xc000056fc0 sp=0xc000056f70 pc=0x4e4c44
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000056fe0 sp=0xc000056fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000056fe8 sp=0xc000056fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 7 [chan receive]:
runtime.gopark(0x100000000000040?, 0x0?, 0x3?, 0x0?, 0xc0001001d8?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00005dd50 sp=0xc00005dd30 pc=0x4392b6
runtime.chanrecv(0xc000072fc0, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00005dde0 sp=0xc00005dd50 pc=0x40793d
runtime.chanrecv1(0xc0001002a0?, 0x971360?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00005de08 sp=0xc00005dde0 pc=0x407478
testing.tRunner.func1()
        /tmp/workdir/go/src/testing/testing.go:1467 +0x4a5 fp=0xc00005df70 sp=0xc00005de08 pc=0x4e5165
testing.tRunner(0xc000134b60, 0x78dcd8)
        /tmp/workdir/go/src/testing/testing.go:1507 +0x144 fp=0xc00005dfc0 sp=0xc00005df70 pc=0x4e4c44
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00005dfe0 sp=0xc00005dfc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00005dfe8 sp=0xc00005dfe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 147 [semacquire]:
runtime.gopark(0xc000082340?, 0x2?, 0x10?, 0x0?, 0x40e787?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00026dcb8 sp=0xc00026dc98 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x80?, 0x0, 0x0, 0x30?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc00026dd20 sp=0xc00026dcb8 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc00026dd80 sp=0xc00026dd20 pc=0x41a56f
runtime.gcAssistAlloc(0xc0001881a0)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc00026dde0 sp=0xc00026dd80 pc=0x41e15c
runtime.mallocgc(0x8000, 0x6f5280, 0x1)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc00026de50 sp=0xc00026dde0 pc=0x40deda
runtime.makeslice(0xc0001b1790?, 0x7d1e60?, 0xc000330020?)
        /tmp/workdir/go/src/runtime/slice.go:103 +0x52 fp=0xc00026de78 sp=0xc00026de50 pc=0x44dad2
io.copyBuffer({0x7d1e60, 0xc000330020}, {0x7d1d20, 0xc000346020}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:424 +0x117 fp=0xc00026def8 sp=0xc00026de78 pc=0x48fe37
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc00026df58 sp=0xc00026def8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0xc0000c3440?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc00026dfc8 sp=0xc00026df58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc00026dfe0 sp=0xc00026dfc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00026dfe8 sp=0xc00026dfe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 58 [semacquire]:
runtime.gopark(0x4393a0?, 0xc0001881a0?, 0xa0?, 0xc8?, 0xc00039ebb0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00039eb70 sp=0xc00039eb50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x0?, 0x0, 0x0, 0x50?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc00039ebd8 sp=0xc00039eb70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc00039ec38 sp=0xc00039ebd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc0000d4340)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc00039ec98 sp=0xc00039ec38 pc=0x41e15c
runtime.mallocgc(0x400, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc00039ed08 sp=0xc00039ec98 pc=0x40deda
runtime.growslice(0x0, 0x200?, 0xc00023e318?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/slice.go:264 +0x4a8 fp=0xc00039ed68 sp=0xc00039ed08 pc=0x44e008
bytes.growSlice({0xc000408000, 0x17, 0x0?}, 0x0?)
        /tmp/workdir/go/src/bytes/buffer.go:240 +0x9b fp=0xc00039ede8 sp=0xc00039ed68 pc=0x4d167b
bytes.(*Buffer).grow(0xc00022e120, 0x200)
        /tmp/workdir/go/src/bytes/buffer.go:142 +0x14f fp=0xc00039ee20 sp=0xc00039ede8 pc=0x4d10ef
bytes.(*Buffer).ReadFrom(0xc00022e120, {0x7d1d20, 0xc000238040})
        /tmp/workdir/go/src/bytes/buffer.go:200 +0x45 fp=0xc00039ee78 sp=0xc00039ee20 pc=0x4d1485
io.copyBuffer({0x7d1740, 0xc00022e120}, {0x7d1d20, 0xc000238040}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:413 +0x14b fp=0xc00039eef8 sp=0xc00039ee78 pc=0x48fe6b
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc00039ef58 sp=0xc00039eef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0xc0000c3500?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc00039efc8 sp=0xc00039ef58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc00039efe0 sp=0xc00039efc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00039efe8 sp=0xc00039efe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 8 [chan receive]:
runtime.gopark(0xc000308000?, 0xc0003421f8?, 0x80?, 0x53?, 0x473eeb?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000268d40 sp=0xc000268d20 pc=0x4392b6
runtime.chanrecv(0xc000348600, 0xc000268e48, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000268dd0 sp=0xc000268d40 pc=0x40793d
runtime.chanrecv1(0xc0003500f0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000268df8 sp=0xc000268dd0 pc=0x407478
os/exec.(*Cmd).Wait(0xc000332840)
        /tmp/workdir/go/src/os/exec/exec.go:741 +0x16d fp=0xc000268e68 sp=0xc000268df8 pc=0x4f818d
os/exec.(*Cmd).Run(0xc000330100?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc000268e88 sp=0xc000268e68 pc=0x4f6f79
os/exec.(*Cmd).Output(0xc000332840)
        /tmp/workdir/go/src/os/exec/exec.go:767 +0xc7 fp=0xc000268ed8 sp=0xc000268e88 pc=0x4f83a7
os/exec_test.TestCatStdin(0xc0001351e0)
        /tmp/workdir/go/src/os/exec/exec_test.go:343 +0xc5 fp=0xc000268f70 sp=0xc000268ed8 pc=0x6ca6e5
testing.tRunner(0xc0001351e0, 0x78dc78)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000268fc0 sp=0xc000268f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000268fe0 sp=0xc000268fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000268fe8 sp=0xc000268fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 10 [syscall]:
syscall.Syscall6(0x1e1, 0x1, 0x4d1b, 0xc0003a2d68, 0x10020, 0x0, 0x0)
        /tmp/workdir/go/src/syscall/asm_unix_amd64.s:39 +0x5 fp=0xc0003a2d18 sp=0xc0003a2d10 pc=0x478245
os.wait6(...)
        /tmp/workdir/go/src/os/wait6_netbsd.go:15
os.(*Process).blockUntilWaitable(0xc0000222a0)
        /tmp/workdir/go/src/os/wait_wait6.go:20 +0x69 fp=0xc0003a2d90 sp=0xc0003a2d18 pc=0x49d6e9
os.(*Process).wait(0xc0000222a0)
        /tmp/workdir/go/src/os/exec_unix.go:22 +0x28 fp=0xc0003a2df0 sp=0xc0003a2d90 pc=0x497d08
os.(*Process).Wait(...)
        /tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0xc0003909a0)
        /tmp/workdir/go/src/os/exec/exec.go:721 +0x4b fp=0xc0003a2e60 sp=0xc0003a2df0 pc=0x4f806b
os/exec.(*Cmd).Run(0xc00004b740?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc0003a2e80 sp=0xc0003a2e60 pc=0x4f6f79
os/exec.(*Cmd).CombinedOutput(0xc0003909a0)
        /tmp/workdir/go/src/os/exec/exec.go:788 +0xa8 fp=0xc0003a2ea8 sp=0xc0003a2e80 pc=0x4f85a8
os/exec_test.TestCatGoodAndBadFile(0xc000135860)
        /tmp/workdir/go/src/os/exec/exec_test.go:379 +0xaa fp=0xc0003a2f70 sp=0xc0003a2ea8 pc=0x6cabea
testing.tRunner(0xc000135860, 0x78dc70)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0003a2fc0 sp=0xc0003a2f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0003a2fe0 sp=0xc0003a2fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0003a2fe8 sp=0xc0003a2fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 13 [semacquire]:
runtime.gopark(0xc00005cc68?, 0x419640?, 0xa0?, 0xcc?, 0x7f7fceacbc60?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00005cc48 sp=0xc00005cc28 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x0?, 0x0, 0x0, 0x0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc00005ccb0 sp=0xc00005cc48 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc00005cd10 sp=0xc00005ccb0 pc=0x41a56f
runtime.gcAssistAlloc(0xc0001604e0)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc00005cd70 sp=0xc00005cd10 pc=0x41e15c
runtime.mallocgc(0x90, 0x74c0c0, 0x1)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc00005cde0 sp=0xc00005cd70 pc=0x40deda
runtime.newobject(0xc0000aa164?)
        /tmp/workdir/go/src/runtime/malloc.go:1199 +0x27 fp=0xc00005ce08 sp=0xc00005cde0 pc=0x40e787
os.(*Process).wait(0xc0000aa150)
        /tmp/workdir/go/src/os/exec_unix.go:38 +0x6f fp=0xc00005ce68 sp=0xc00005ce08 pc=0x497d4f
os.(*Process).Wait(...)
        /tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0xc0004f0840)
        /tmp/workdir/go/src/os/exec/exec.go:721 +0x4b fp=0xc00005ced8 sp=0xc00005ce68 pc=0x4f806b
os/exec.(*Cmd).Run(0xc000160340?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc00005cef8 sp=0xc00005ced8 pc=0x4f6f79
os/exec_test.TestExitCode(0xc000160340)
        /tmp/workdir/go/src/os/exec/exec_test.go:452 +0x225 fp=0xc00005cf70 sp=0xc00005cef8 pc=0x6cb305
testing.tRunner(0xc000160340, 0x78dcc8)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc00005cfc0 sp=0xc00005cf70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00005cfe0 sp=0xc00005cfc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00005cfe8 sp=0xc00005cfe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 14 [runnable]:
runtime.gcTrigger.test({0x0?, 0x0?, 0x0?})
        /tmp/workdir/go/src/runtime/mgc.go:547 +0xfb fp=0xc000198bb0 sp=0xc000198ba8 pc=0x419edb
runtime.mallocgc(0x90, 0x74c0c0, 0x1)
        /tmp/workdir/go/src/runtime/malloc.go:1144 +0x7e5 fp=0xc000198c20 sp=0xc000198bb0 pc=0x40e5a5
runtime.newobject(0xc0000221f4?)
        /tmp/workdir/go/src/runtime/malloc.go:1199 +0x27 fp=0xc000198c48 sp=0xc000198c20 pc=0x40e787
os.(*Process).wait(0xc0000221e0)
        /tmp/workdir/go/src/os/exec_unix.go:38 +0x6f fp=0xc000198ca8 sp=0xc000198c48 pc=0x497d4f
os.(*Process).Wait(...)
        /tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0xc000390160)
        /tmp/workdir/go/src/os/exec/exec.go:721 +0x4b fp=0xc000198d18 sp=0xc000198ca8 pc=0x4f806b
os/exec_test.TestPipes(0xc000160680)
        /tmp/workdir/go/src/os/exec/exec_test.go:528 +0xb93 fp=0xc000198f70 sp=0xc000198d18 pc=0x6cc0f3
testing.tRunner(0xc000160680, 0x78dd48)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000198fc0 sp=0xc000198f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000198fe0 sp=0xc000198fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000198fe8 sp=0xc000198fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 15 [chan receive]:
runtime.gopark(0x4392b6?, 0xc0000289b0?, 0x0?, 0x60?, 0xc00018d5e8?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00018d540 sp=0xc00018d520 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00018d5d0 sp=0xc00018d540 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00018d5f8 sp=0xc00018d5d0 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc00018d620 sp=0xc00018d5f8 pc=0x4e5bcb
testing.(*T).Parallel(0xc0001609c0)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc00018d688 sp=0xc00018d620 pc=0x4e4930
os/exec_test.TestStdinClose(0xc0001609c0)
        /tmp/workdir/go/src/os/exec/exec_test.go:536 +0x47 fp=0xc00018d770 sp=0xc00018d688 pc=0x6cc327
testing.tRunner(0xc0001609c0, 0x78dd58)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc00018d7c0 sp=0xc00018d770 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00018d7e0 sp=0xc00018d7c0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00018d7e8 sp=0xc00018d7e0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 16 [chan receive]:
runtime.gopark(0x4392b6?, 0xc0000289b0?, 0x0?, 0x60?, 0xc00018de40?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00018dd98 sp=0xc00018dd78 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00018de28 sp=0xc00018dd98 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00018de50 sp=0xc00018de28 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc00018de78 sp=0xc00018de50 pc=0x4e5bcb
testing.(*T).Parallel(0xc000160d00)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc00018dee0 sp=0xc00018de78 pc=0x4e4930
os/exec_test.TestStdinCloseRace(0xc000160d00)
        /tmp/workdir/go/src/os/exec/exec_test.go:579 +0x45 fp=0xc00018df70 sp=0xc00018dee0 pc=0x6ccb65
testing.tRunner(0xc000160d00, 0x78dd50)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc00018dfc0 sp=0xc00018df70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00018dfe0 sp=0xc00018dfc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00018dfe8 sp=0xc00018dfe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 75 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc000192d78?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000192cd0 sp=0xc000192cb0 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000192d60 sp=0xc000192cd0 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000192d88 sp=0xc000192d60 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000192db0 sp=0xc000192d88 pc=0x4e5bcb
testing.(*T).Parallel(0xc000188340)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000192e18 sp=0xc000192db0 pc=0x4e4930
os/exec_test.TestExtraFilesRace(0xc000188340)
        /tmp/workdir/go/src/os/exec/exec_test.go:794 +0x36 fp=0xc000192f70 sp=0xc000192e18 pc=0x6ce6b6
testing.tRunner(0xc000188340, 0x78dcf0)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000192fc0 sp=0xc000192f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000192fe0 sp=0xc000192fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000192fe8 sp=0xc000192fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 35 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc0004d2e20?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d2d78 sp=0xc0004d2d58 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc0004d2e08 sp=0xc0004d2d78 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc0004d2e30 sp=0xc0004d2e08 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0004d2e58 sp=0xc0004d2e30 pc=0x4e5bcb
testing.(*T).Parallel(0xc0004d0680)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc0004d2ec0 sp=0xc0004d2e58 pc=0x4e4930
os/exec_test.TestStringPathNotResolved(0xc0004d0680)
        /tmp/workdir/go/src/os/exec/exec_test.go:1052 +0x33 fp=0xc0004d2f70 sp=0xc0004d2ec0 pc=0x6d03b3
testing.tRunner(0xc0004d0680, 0x78dd60)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0004d2fc0 sp=0xc0004d2f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0004d2fe0 sp=0xc0004d2fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d2fe8 sp=0xc0004d2fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 146 [semacquire]:
runtime.gopark(0xc0000ac9c0?, 0xc000045430?, 0xb0?, 0x67?, 0x427814?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00026cb70 sp=0xc00026cb50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x8b?, 0x0, 0x0, 0x4?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc00026cbd8 sp=0xc00026cb70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc00026cc38 sp=0xc00026cbd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc000161a00)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc00026cc98 sp=0xc00026cc38 pc=0x41e15c
runtime.mallocgc(0x200, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc00026cd08 sp=0xc00026cc98 pc=0x40deda
runtime.growslice(0x0, 0xc00023e3c0?, 0xc0000455b0?, 0x440865?, 0xc000161a00?)
        /tmp/workdir/go/src/runtime/slice.go:264 +0x4a8 fp=0xc00026cd68 sp=0xc00026cd08 pc=0x44e008
bytes.growSlice({0x0, 0x0, 0x40b1ad?}, 0xc000045668?)
        /tmp/workdir/go/src/bytes/buffer.go:240 +0x9b fp=0xc00026cde8 sp=0xc00026cd68 pc=0x4d167b
bytes.(*Buffer).grow(0xc000324060, 0x200)
        /tmp/workdir/go/src/bytes/buffer.go:142 +0x14f fp=0xc00026ce20 sp=0xc00026cde8 pc=0x4d10ef
bytes.(*Buffer).ReadFrom(0xc000324060, {0x7d1d20, 0xc000346008})
        /tmp/workdir/go/src/bytes/buffer.go:200 +0x45 fp=0xc00026ce78 sp=0xc00026ce20 pc=0x4d1485
io.copyBuffer({0x7d1740, 0xc000324060}, {0x7d1d20, 0xc000346008}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:413 +0x14b fp=0xc00026cef8 sp=0xc00026ce78 pc=0x48fe6b
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc00026cf58 sp=0xc00026cef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0x78dc88?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc00026cfc8 sp=0xc00026cf58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc00026cfe0 sp=0xc00026cfc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00026cfe8 sp=0xc00026cfe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 82 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d6750 sp=0xc0004d6730 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc0004d67e0 sp=0xc0004d6750 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d67e8 sp=0xc0004d67e0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 83 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d6f50 sp=0xc0004d6f30 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc0004d6fe0 sp=0xc0004d6f50 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d6fe8 sp=0xc0004d6fe0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 67 [GC worker (idle)]:
runtime.gopark(0xc00018ef80?, 0xc00001a890?, 0x0?, 0x0?, 0x5f8f00?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00018ef50 sp=0xc00018ef30 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc00018efe0 sp=0xc00018ef50 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00018efe8 sp=0xc00018efe0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 68 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000045f50 sp=0xc000045f30 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc000045fe0 sp=0xc000045f50 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000045fe8 sp=0xc000045fe0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 69 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000046750 sp=0xc000046730 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc0000467e0 sp=0xc000046750 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000467e8 sp=0xc0000467e0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 84 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d7750 sp=0xc0004d7730 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc0004d77e0 sp=0xc0004d7750 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d77e8 sp=0xc0004d77e0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 85 [GC worker (idle)]:
runtime.gopark(0x1980c5fe018?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d7f50 sp=0xc0004d7f30 pc=0x4392b6
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1266 +0xf1 fp=0xc0004d7fe0 sp=0xc0004d7f50 pc=0x41b7b1
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d7fe8 sp=0xc0004d7fe0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 70 [GC mark termination]:
runtime.systemstack_switch()
        /tmp/workdir/go/src/runtime/asm_amd64.s:459 fp=0xc000046ef0 sp=0xc000046ee8 pc=0x4679e0
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:807 +0xff fp=0xc000046f50 sp=0xc000046ef0 pc=0x41a63f
runtime.gcBgMarkWorker()
        /tmp/workdir/go/src/runtime/mgc.go:1398 +0x345 fp=0xc000046fe0 sp=0xc000046f50 pc=0x41ba05
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000046fe8 sp=0xc000046fe0 pc=0x469c01
created by runtime.gcBgMarkStartWorkers
        /tmp/workdir/go/src/runtime/mgc.go:1190 +0x25

goroutine 76 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc000193688?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0001935e0 sp=0xc0001935c0 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000193670 sp=0xc0001935e0 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000193698 sp=0xc000193670 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0001936c0 sp=0xc000193698 pc=0x4e5bcb
testing.(*T).Parallel(0xc0001884e0)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000193728 sp=0xc0001936c0 pc=0x4e4930
os/exec_test.TestIgnorePipeErrorOnSuccess(0x0?)
        /tmp/workdir/go/src/os/exec/exec_test.go:862 +0x25 fp=0xc000193770 sp=0xc000193728 pc=0x6cf105
testing.tRunner(0xc0001884e0, 0x78dd08)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0001937c0 sp=0xc000193770 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0001937e0 sp=0xc0001937c0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001937e8 sp=0xc0001937e0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 77 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0x39?, 0xc000193e90?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000193de8 sp=0xc000193dc8 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000193e78 sp=0xc000193de8 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000193ea0 sp=0xc000193e78 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000193ec8 sp=0xc000193ea0 pc=0x4e5bcb
testing.(*T).Parallel(0xc000188820)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000193f30 sp=0xc000193ec8 pc=0x4e4930
os/exec_test.TestClosePipeOnCopyError(0xc000188820)
        /tmp/workdir/go/src/os/exec/exec_test.go:891 +0x25 fp=0xc000193f70 sp=0xc000193f30 pc=0x6cf245
testing.tRunner(0xc000188820, 0x78dc80)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000193fc0 sp=0xc000193f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000193fe0 sp=0xc000193fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000193fe8 sp=0xc000193fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 40 [semacquire]:
runtime.gopark(0x473cf3?, 0x476e2e?, 0x7e?, 0xa4?, 0x9?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000235b70 sp=0xc000235b50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x30?, 0x0, 0x0, 0xd0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000235bd8 sp=0xc000235b70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000235c38 sp=0xc000235bd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc000188d00)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000235c98 sp=0xc000235c38 pc=0x41e15c
runtime.mallocgc(0x200, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000235d08 sp=0xc000235c98 pc=0x40deda
runtime.growslice(0x0, 0x6f3f00?, 0x78dc40?, 0x11?, 0x11?)
        /tmp/workdir/go/src/runtime/slice.go:264 +0x4a8 fp=0xc000235d68 sp=0xc000235d08 pc=0x44e008
bytes.growSlice({0x0, 0x0, 0x40b1ad?}, 0x7263e0?)
        /tmp/workdir/go/src/bytes/buffer.go:240 +0x9b fp=0xc000235de8 sp=0xc000235d68 pc=0x4d167b
bytes.(*Buffer).grow(0xc0003941b0, 0x200)
        /tmp/workdir/go/src/bytes/buffer.go:142 +0x14f fp=0xc000235e20 sp=0xc000235de8 pc=0x4d10ef
bytes.(*Buffer).ReadFrom(0xc0003941b0, {0x7d1d20, 0xc0002ce088})
        /tmp/workdir/go/src/bytes/buffer.go:200 +0x45 fp=0xc000235e78 sp=0xc000235e20 pc=0x4d1485
io.copyBuffer({0x7d1740, 0xc0003941b0}, {0x7d1d20, 0xc0002ce088}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:413 +0x14b fp=0xc000235ef8 sp=0xc000235e78 pc=0x48fe6b
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc000235f58 sp=0xc000235ef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0x78dc70?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc000235fc8 sp=0xc000235f58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc000235fe0 sp=0xc000235fc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000235fe8 sp=0xc000235fe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 79 [semacquire]:
runtime.gopark(0xc000230c20?, 0x419640?, 0x58?, 0xc?, 0x7f7ff7ac4500?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000230c00 sp=0xc000230be0 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x0?, 0x0, 0x0, 0x11?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000230c68 sp=0xc000230c00 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000230cc8 sp=0xc000230c68 pc=0x41a56f
runtime.gcAssistAlloc(0xc000189040)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000230d28 sp=0xc000230cc8 pc=0x41e15c
runtime.mallocgc(0x90, 0x74c0c0, 0x1)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000230d98 sp=0xc000230d28 pc=0x40deda
runtime.newobject(0xc0002ec074?)
        /tmp/workdir/go/src/runtime/malloc.go:1199 +0x27 fp=0xc000230dc0 sp=0xc000230d98 pc=0x40e787
os.(*Process).wait(0xc0002ec060)
        /tmp/workdir/go/src/os/exec_unix.go:38 +0x6f fp=0xc000230e20 sp=0xc000230dc0 pc=0x497d4f
os.(*Process).Wait(...)
        /tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0xc0002c0000)
        /tmp/workdir/go/src/os/exec/exec.go:721 +0x4b fp=0xc000230e90 sp=0xc000230e20 pc=0x4f806b
os/exec_test.TestContext(0xc000188ea0)
        /tmp/workdir/go/src/os/exec/exec_test.go:944 +0x371 fp=0xc000230f70 sp=0xc000230e90 pc=0x6cf811
testing.tRunner(0xc000188ea0, 0x78dc98)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000230fc0 sp=0xc000230f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000230fe0 sp=0xc000230fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000230fe8 sp=0xc000230fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 80 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc0004d8e00?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d8d58 sp=0xc0004d8d38 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc0004d8de8 sp=0xc0004d8d58 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc0004d8e10 sp=0xc0004d8de8 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0004d8e38 sp=0xc0004d8e10 pc=0x4e5bcb
testing.(*T).Parallel(0xc0001891e0)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc0004d8ea0 sp=0xc0004d8e38 pc=0x4e4930
os/exec_test.TestContextCancel(0xc0001891e0)
        /tmp/workdir/go/src/os/exec/exec_test.go:957 +0x45 fp=0xc0004d8f70 sp=0xc0004d8ea0 pc=0x6cf8e5
testing.tRunner(0xc0001891e0, 0x78dc90)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0004d8fc0 sp=0xc0004d8f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0004d8fe0 sp=0xc0004d8fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d8fe8 sp=0xc0004d8fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 81 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc0004d9618?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d9570 sp=0xc0004d9550 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc0004d9600 sp=0xc0004d9570 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc0004d9628 sp=0xc0004d9600 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0004d9650 sp=0xc0004d9628 pc=0x4e5bcb
testing.(*T).Parallel(0xc000189520)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc0004d96b8 sp=0xc0004d9650 pc=0x4e4930
os/exec_test.TestDedupEnvEcho(0xc000189520)
        /tmp/workdir/go/src/os/exec/exec_test.go:1014 +0x33 fp=0xc0004d9770 sp=0xc0004d96b8 pc=0x6cfd13
testing.tRunner(0xc000189520, 0x78dca8)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0004d97c0 sp=0xc0004d9770 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0004d97e0 sp=0xc0004d97c0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d97e8 sp=0xc0004d97e0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 98 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc00019ac88?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00019abe0 sp=0xc00019abc0 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00019ac70 sp=0xc00019abe0 pc=0x40793d
runtime.chanrecv1(0xc0001001e0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00019ac98 sp=0xc00019ac70 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc00019acc0 sp=0xc00019ac98 pc=0x4e5bcb
testing.(*T).Parallel(0xc000189860)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc00019ad28 sp=0xc00019acc0 pc=0x4e4930
os/exec_test.TestString(0xc000189860)
        /tmp/workdir/go/src/os/exec/exec_test.go:1028 +0x36 fp=0xc00019af70 sp=0xc00019ad28 pc=0x6cff96
testing.tRunner(0xc000189860, 0x78dd68)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc00019afc0 sp=0xc00019af70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00019afe0 sp=0xc00019afc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00019afe8 sp=0xc00019afe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 106 [chan receive]:
runtime.gopark(0x4392b6?, 0x7ce5e8?, 0x0?, 0xa5?, 0xc0004d9d70?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004d9cc8 sp=0xc0004d9ca8 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc0004d9d58 sp=0xc0004d9cc8 pc=0x40793d
runtime.chanrecv1(0xc0001002a0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc0004d9d80 sp=0xc0004d9d58 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0004d9da8 sp=0xc0004d9d80 pc=0x4e5bcb
testing.(*T).Parallel(0xc0000836c0)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc0004d9e10 sp=0xc0004d9da8 pc=0x4e4930
os/exec_test.TestExplicitPWD.func1(0xc0000836c0)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:243 +0x3e fp=0xc0004d9f70 sp=0xc0004d9e10 pc=0x6c837e
testing.tRunner(0xc0000836c0, 0xc0000c3600)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0004d9fc0 sp=0xc0004d9f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0004d9fe0 sp=0xc0004d9fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004d9fe8 sp=0xc0004d9fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 100 [chan receive]:
runtime.gopark(0xc00004d800?, 0xc0003420d8?, 0x40?, 0x23?, 0x473eeb?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00019ccb0 sp=0xc00019cc90 pc=0x4392b6
runtime.chanrecv(0xc0003481e0, 0xc00019cdb8, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc00019cd40 sp=0xc00019ccb0 pc=0x40793d
runtime.chanrecv1(0xc000350060?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc00019cd68 sp=0xc00019cd40 pc=0x407478
os/exec.(*Cmd).Wait(0xc000332000)
        /tmp/workdir/go/src/os/exec/exec.go:741 +0x16d fp=0xc00019cdd8 sp=0xc00019cd68 pc=0x4f818d
os/exec.(*Cmd).Run(0xc000330020?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc00019cdf8 sp=0xc00019cdd8 pc=0x4f6f79
os/exec.(*Cmd).Output(0xc000332000)
        /tmp/workdir/go/src/os/exec/exec.go:767 +0xc7 fp=0xc00019ce48 sp=0xc00019cdf8 pc=0x4f83a7
os/exec_test.TestImplicitPWD.func1(0xc000007d40)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:192 +0x430 fp=0xc00019cf70 sp=0xc00019ce48 pc=0x6c7d50
testing.tRunner(0xc000007d40, 0xc0000c3440)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc00019cfc0 sp=0xc00019cf70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc00019cfe0 sp=0xc00019cfc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00019cfe8 sp=0xc00019cfe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 101 [chan receive]:
runtime.gopark(0xc0004fa000?, 0xc00044a258?, 0x20?, 0x28?, 0x473eeb?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0003a3cb0 sp=0xc0003a3c90 pc=0x4392b6
runtime.chanrecv(0xc0002d4660, 0xc0003a3db8, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc0003a3d40 sp=0xc0003a3cb0 pc=0x40793d
runtime.chanrecv1(0xc0002ec180?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc0003a3d68 sp=0xc0003a3d40 pc=0x407478
os/exec.(*Cmd).Wait(0xc0002c0840)
        /tmp/workdir/go/src/os/exec/exec.go:741 +0x16d fp=0xc0003a3dd8 sp=0xc0003a3d68 pc=0x4f818d
os/exec.(*Cmd).Run(0xc0000681e0?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc0003a3df8 sp=0xc0003a3dd8 pc=0x4f6f79
os/exec.(*Cmd).Output(0xc0002c0840)
        /tmp/workdir/go/src/os/exec/exec.go:767 +0xc7 fp=0xc0003a3e48 sp=0xc0003a3df8 pc=0x4f83a7
os/exec_test.TestImplicitPWD.func1(0xc000082680)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:192 +0x430 fp=0xc0003a3f70 sp=0xc0003a3e48 pc=0x6c7d50
testing.tRunner(0xc000082680, 0xc0000c3480)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0003a3fc0 sp=0xc0003a3f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0003a3fe0 sp=0xc0003a3fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0003a3fe8 sp=0xc0003a3fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 103 [chan receive]:
runtime.gopark(0x9a8a80?, 0xc00048e000?, 0xe0?, 0x31?, 0x473eeb?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000234cb0 sp=0xc000234c90 pc=0x4392b6
runtime.chanrecv(0xc00023e480, 0xc000234db8, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000234d40 sp=0xc000234cb0 pc=0x40793d
runtime.chanrecv1(0xc0002460c0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000234d68 sp=0xc000234d40 pc=0x407478
os/exec.(*Cmd).Wait(0xc00023a2c0)
        /tmp/workdir/go/src/os/exec/exec.go:741 +0x16d fp=0xc000234dd8 sp=0xc000234d68 pc=0x4f818d
os/exec.(*Cmd).Run(0xc00009c0e0?)
        /tmp/workdir/go/src/os/exec/exec.go:500 +0x39 fp=0xc000234df8 sp=0xc000234dd8 pc=0x4f6f79
os/exec.(*Cmd).Output(0xc00023a2c0)
        /tmp/workdir/go/src/os/exec/exec.go:767 +0xc7 fp=0xc000234e48 sp=0xc000234df8 pc=0x4f83a7
os/exec_test.TestImplicitPWD.func1(0xc000082ea0)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:192 +0x430 fp=0xc000234f70 sp=0xc000234e48 pc=0x6c7d50
testing.tRunner(0xc000082ea0, 0xc0000c3500)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000234fc0 sp=0xc000234f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000234fe0 sp=0xc000234fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000234fe8 sp=0xc000234fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 107 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc000480d70?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000480cc8 sp=0xc000480ca8 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000480d58 sp=0xc000480cc8 pc=0x40793d
runtime.chanrecv1(0xc0001002a0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000480d80 sp=0xc000480d58 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000480da8 sp=0xc000480d80 pc=0x4e5bcb
testing.(*T).Parallel(0xc000083860)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000480e10 sp=0xc000480da8 pc=0x4e4930
os/exec_test.TestExplicitPWD.func1(0xc000083860)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:243 +0x3e fp=0xc000480f70 sp=0xc000480e10 pc=0x6c837e
testing.tRunner(0xc000083860, 0xc0000c3640)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000480fc0 sp=0xc000480f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000480fe0 sp=0xc000480fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000480fe8 sp=0xc000480fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 108 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc000190d70?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000190cc8 sp=0xc000190ca8 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000190d58 sp=0xc000190cc8 pc=0x40793d
runtime.chanrecv1(0xc0001002a0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000190d80 sp=0xc000190d58 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc000190da8 sp=0xc000190d80 pc=0x4e5bcb
testing.(*T).Parallel(0xc000083a00)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000190e10 sp=0xc000190da8 pc=0x4e4930
os/exec_test.TestExplicitPWD.func1(0xc000083a00)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:243 +0x3e fp=0xc000190f70 sp=0xc000190e10 pc=0x6c837e
testing.tRunner(0xc000083a00, 0xc0000c3680)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc000190fc0 sp=0xc000190f70 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc000190fe0 sp=0xc000190fc0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000190fe8 sp=0xc000190fe0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f

goroutine 133 [semacquire]:
runtime.gopark(0xc0000474d0?, 0xc000047520?, 0x32?, 0x82?, 0x47710d?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000237cb8 sp=0xc000237c98 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0xf1?, 0x0, 0x0, 0x0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000237d20 sp=0xc000237cb8 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000237d80 sp=0xc000237d20 pc=0x41a56f
runtime.gcAssistAlloc(0xc0004d0d00)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000237de0 sp=0xc000237d80 pc=0x41e15c
runtime.mallocgc(0x8000, 0x6f5280, 0x1)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000237e50 sp=0xc000237de0 pc=0x40deda
runtime.makeslice(0x7f7fceacb878?, 0x7d1e60?, 0xc0000681e0?)
        /tmp/workdir/go/src/runtime/slice.go:103 +0x52 fp=0xc000237e78 sp=0xc000237e50 pc=0x44dad2
io.copyBuffer({0x7d1e60, 0xc0000681e0}, {0x7d1d20, 0xc0002cc078}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:424 +0x117 fp=0xc000237ef8 sp=0xc000237e78 pc=0x48fe37
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc000237f58 sp=0xc000237ef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0x78dd30?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc000237fc8 sp=0xc000237f58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc000237fe0 sp=0xc000237fc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000237fe8 sp=0xc000237fe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 149 [semacquire]:
runtime.gopark(0x9ae040?, 0x425e45?, 0xe6?, 0x79?, 0x2000?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000231b70 sp=0xc000231b50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0xe7?, 0x0, 0x0, 0x0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000231bd8 sp=0xc000231b70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000231c38 sp=0xc000231bd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc0004d0ea0)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000231c98 sp=0xc000231c38 pc=0x41e15c
runtime.mallocgc(0x200, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000231d08 sp=0xc000231c98 pc=0x40deda
runtime.growslice(0x0, 0xc0003923c0?, 0xc0004d35b0?, 0x440865?, 0xc0004d0ea0?)
        /tmp/workdir/go/src/runtime/slice.go:264 +0x4a8 fp=0xc000231d68 sp=0xc000231d08 pc=0x44e008
bytes.growSlice({0x0, 0x0, 0x40b1ad?}, 0xc0004d3668?)
        /tmp/workdir/go/src/bytes/buffer.go:240 +0x9b fp=0xc000231de8 sp=0xc000231d68 pc=0x4d167b
bytes.(*Buffer).grow(0xc0003240f0, 0x200)
        /tmp/workdir/go/src/bytes/buffer.go:142 +0x14f fp=0xc000231e20 sp=0xc000231de8 pc=0x4d10ef
bytes.(*Buffer).ReadFrom(0xc0003240f0, {0x7d1d20, 0xc000346060})
        /tmp/workdir/go/src/bytes/buffer.go:200 +0x45 fp=0xc000231e78 sp=0xc000231e20 pc=0x4d1485
io.copyBuffer({0x7d1740, 0xc0003240f0}, {0x7d1d20, 0xc000346060}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:413 +0x14b fp=0xc000231ef8 sp=0xc000231e78 pc=0x48fe6b
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc000231f58 sp=0xc000231ef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0x78dcb0?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc000231fc8 sp=0xc000231f58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc000231fe0 sp=0xc000231fc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000231fe8 sp=0xc000231fe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 132 [semacquire]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000236b70 sp=0xc000236b50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0x0?, 0x0, 0x0, 0x0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000236bd8 sp=0xc000236b70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000236c38 sp=0xc000236bd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc000404000)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000236c98 sp=0xc000236c38 pc=0x41e15c
runtime.mallocgc(0x200, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000236d08 sp=0xc000236c98 pc=0x40deda
runtime.growslice(0x0, 0x3f4?, 0xc00024cdb0?, 0x440865?, 0xc000404000?)
        /tmp/workdir/go/src/runtime/slice.go:264 +0x4a8 fp=0xc000236d68 sp=0xc000236d08 pc=0x44e008
bytes.growSlice({0x0, 0x0, 0x40b1ad?}, 0x465c7d?)
        /tmp/workdir/go/src/bytes/buffer.go:240 +0x9b fp=0xc000236de8 sp=0xc000236d68 pc=0x4d167b
bytes.(*Buffer).grow(0xc0003940c0, 0x200)
        /tmp/workdir/go/src/bytes/buffer.go:142 +0x14f fp=0xc000236e20 sp=0xc000236de8 pc=0x4d10ef
bytes.(*Buffer).ReadFrom(0xc0003940c0, {0x7d1d20, 0xc0002cc060})
        /tmp/workdir/go/src/bytes/buffer.go:200 +0x45 fp=0xc000236e78 sp=0xc000236e20 pc=0x4d1485
io.copyBuffer({0x7d1740, 0xc0003940c0}, {0x7d1d20, 0xc0002cc060}, {0x0, 0x0, 0x0})
        /tmp/workdir/go/src/io/io.go:413 +0x14b fp=0xc000236ef8 sp=0xc000236e78 pc=0x48fe6b
io.Copy(...)
        /tmp/workdir/go/src/io/io.go:386
os/exec.(*Cmd).writerDescriptor.func1()
        /tmp/workdir/go/src/os/exec/exec.go:470 +0x3a fp=0xc000236f58 sp=0xc000236ef8 pc=0x4f6efa
os/exec.(*Cmd).Start.func2(0x0?)
        /tmp/workdir/go/src/os/exec/exec.go:624 +0x32 fp=0xc000236fc8 sp=0xc000236f58 pc=0x4f7bd2
os/exec.(*Cmd).Start.func3()
        /tmp/workdir/go/src/os/exec/exec.go:636 +0x2a fp=0xc000236fe0 sp=0xc000236fc8 pc=0x4f7b6a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000236fe8 sp=0xc000236fe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:623 +0x9bf

goroutine 130 [chan send]:
runtime.gopark(0x4769b8?, 0x25?, 0x1e?, 0x64?, 0x9?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000047e50 sp=0xc000047e30 pc=0x4392b6
runtime.chansend(0xc0002a2060, 0xc000047f50, 0x1, 0xc000047ee0?)
        /tmp/workdir/go/src/runtime/chan.go:259 +0x42e fp=0xc000047ed8 sp=0xc000047e50 pc=0x406b6e
runtime.chansend1(0xc000047f70?, 0x7d3d00?)
        /tmp/workdir/go/src/runtime/chan.go:145 +0x1d fp=0xc000047f08 sp=0xc000047ed8 pc=0x40671d
os/exec.(*Cmd).watchCtx(0xc0002c0000, 0xc0002a2060)
        /tmp/workdir/go/src/os/exec/exec.go:672 +0x188 fp=0xc000047fc0 sp=0xc000047f08 pc=0x4f7f88
os/exec.(*Cmd).Start.func4()
        /tmp/workdir/go/src/os/exec/exec.go:644 +0x2a fp=0xc000047fe0 sp=0xc000047fc0 pc=0x4f7b0a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000047fe8 sp=0xc000047fe0 pc=0x469c01
created by os/exec.(*Cmd).Start
        /tmp/workdir/go/src/os/exec/exec.go:644 +0x97f

goroutine 109 [chan receive]:
runtime.gopark(0x4392b6?, 0xc00002aeb0?, 0x0?, 0x39?, 0xc000481570?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc0004814c8 sp=0xc0004814a8 pc=0x4392b6
runtime.chanrecv(0xc000100180, 0x0, 0x1)
        /tmp/workdir/go/src/runtime/chan.go:583 +0x49d fp=0xc000481558 sp=0xc0004814c8 pc=0x40793d
runtime.chanrecv1(0xc0001002a0?, 0x0?)
        /tmp/workdir/go/src/runtime/chan.go:442 +0x18 fp=0xc000481580 sp=0xc000481558 pc=0x407478
testing.(*testContext).waitParallel(0xc00007e1e0)
        /tmp/workdir/go/src/testing/testing.go:1612 +0xab fp=0xc0004815a8 sp=0xc000481580 pc=0x4e5bcb
testing.(*T).Parallel(0xc0001fc000)
        /tmp/workdir/go/src/testing/testing.go:1314 +0x1d0 fp=0xc000481610 sp=0xc0004815a8 pc=0x4e4930
os/exec_test.TestExplicitPWD.func1(0xc0001fc000)
        /tmp/workdir/go/src/os/exec/exec_posix_test.go:243 +0x3e fp=0xc000481770 sp=0xc000481610 pc=0x6c837e
testing.tRunner(0xc0001fc000, 0xc0000c36c0)
        /tmp/workdir/go/src/testing/testing.go:1501 +0x10b fp=0xc0004817c0 sp=0xc000481770 pc=0x4e4c0b
testing.(*T).Run.func1()
        /tmp/workdir/go/src/testing/testing.go:1548 +0x2a fp=0xc0004817e0 sp=0xc0004817c0 pc=0x4e5a8a
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0004817e8 sp=0xc0004817e0 pc=0x469c01
created by testing.(*T).Run
        /tmp/workdir/go/src/testing/testing.go:1548 +0x35f
FAIL    os/exec 10.189s

The signature of this failure appears to be: goroutines blocked in runtime.gopark, goroutines blocked in runtime.goparkunlock, and at least one goroutine blocked in syscall (but not always the wait6 system call specifically).

(attn @golang/netbsd; CC @golang/runtime)

[riastradh]

Thanks for digging into this!

There's a lot of threads tangled up here. Some questions:

1. Do you know which threads appear to have missed a wakeup -- that is, which ones are waiting to be unparked for a condition that has already been satisfied?
2. Do you know what arguments are being passed to wait6 when it's hanging, and what processes are in the system at the time?
3. Can you run crash(8) to get a stack trace for the thread waiting in wait6, using the ps and bt/a commands? (See ddb(4) for information about the commands that crash(8) supports.)
4. What other syscalls than wait6 have you seen in this collection of stack traces? Can you repeat (3) for these too?

[gopherbot]

Found new dashboard test flakes for:

#!watchflakes
post <- goos == "netbsd" && `^\s*runtime\.gopark`

2022-10-12 22:55 netbsd-386-9_0 go@1a7f08cf cmd/compile/internal/test.TestInst (log)

panic: test timed out after 3m0s

syscall.Syscall6(0x1, 0x4cbf, 0x0, 0x10020, 0x0, 0x0, 0x0)
	/tmp/workdir/go/src/syscall/asm_unix_386.s:43 +0x5 fp=0x8c9bdc8 sp=0x8c9bdc4 pc=0x80bf275
os.(*Process).blockUntilWaitable(0x8c1c3c0)
	/tmp/workdir/go/src/os/wait_wait6.go:33 +0x51 fp=0x8c9bdf8 sp=0x8c9bdc8 pc=0x80ed741
os.(*Process).wait(0x8c1c3c0)
	/tmp/workdir/go/src/os/exec_unix.go:22 +0x28 fp=0x8c9be34 sp=0x8c9bdf8 pc=0x80e72f8
os.(*Process).Wait(...)
	/tmp/workdir/go/src/os/exec.go:132
os/exec.(*Cmd).Wait(0x8eec580)
	/tmp/workdir/go/src/os/exec/exec.go:721 +0x46 fp=0x8c9be70 sp=0x8c9be34 pc=0x817ac96
os/exec.(*Cmd).Run(0x8eec580)
	/tmp/workdir/go/src/os/exec/exec.go:500 +0x43 fp=0x8c9be80 sp=0x8c9be70 pc=0x8179dd3
os/exec.(*Cmd).CombinedOutput(0x8eec580)
	/tmp/workdir/go/src/os/exec/exec.go:788 +0x89 fp=0x8c9be94 sp=0x8c9be80 pc=0x817b169
cmd/compile/internal/test.TestInst(0x8eebc20)
	/tmp/workdir/go/src/cmd/compile/internal/test/inst_test.go:58 +0x6fa fp=0x8c9bf9c sp=0x8c9be94 pc=0x83fe5da
testing.tRunner(0x8eebc20, 0x84ae85c)

— watchflakes

[bcmills]

2022-10-12 22:55 netbsd-386-9_0 go@1a7f08cf

Hmm, that one could plausibly be a deadlock in a Go subprocess, which may or may not be related (the test really ought to send SIGQUIT to help debug). I'll update the watchflakes pattern to try to filter it out.

[bcmills]

The other failures that I think are related to this issue are:

• https://storage.googleapis.com/go-build-log/c2fb4188/netbsd-amd64-9_0_06fa9a39.log
• https://storage.googleapis.com/go-build-log/267326a7/netbsd-amd64-9_0_fc356467.log

with lots of stacks like:

runtime.gopark(0xc00049cb38?, 0x4222a5?, 0xa0?, 0x54?, 0x4b097e?)
	/tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc00049caf8 sp=0xc00049cad8 pc=0x439bf6
runtime.goparkunlock(...)
	/tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0xf94eb8, 0x67?, 0x0, 0x0, 0xc0?)
	/tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc00049cb60 sp=0xc00049caf8 pc=0x44ad2f
runtime.semacquire(...)
	/tmp/workdir/go/src/runtime/sema.go:111
runtime.gcStart({0x7f7ff7e9a878?, 0x480317?, 0x49cc10?})
	/tmp/workdir/go/src/runtime/mgc.go:607 +0xf5 fp=0xc00049cbe8 sp=0xc00049cb60 pc=0x41a675
runtime.mallocgc(0x90, 0xb026c0, 0x1)
	/tmp/workdir/go/src/runtime/malloc.go:1145 +0x7f4 fp=0xc00049cc58 sp=0xc00049cbe8 pc=0x40ec34

and the aforementioned local deadlock with stacks like

runtime.gopark(0x9ae040?, 0x425e45?, 0xe6?, 0x79?, 0x2000?)
        /tmp/workdir/go/src/runtime/proc.go:378 +0xd6 fp=0xc000231b70 sp=0xc000231b50 pc=0x4392b6
runtime.goparkunlock(...)
        /tmp/workdir/go/src/runtime/proc.go:384
runtime.semacquire1(0x9a873c, 0xe7?, 0x0, 0x0, 0x0?)
        /tmp/workdir/go/src/runtime/sema.go:160 +0x20f fp=0xc000231bd8 sp=0xc000231b70 pc=0x44a2cf
runtime.semacquire(...)
        /tmp/workdir/go/src/runtime/sema.go:111
runtime.gcMarkDone()
        /tmp/workdir/go/src/runtime/mgc.go:787 +0x2f fp=0xc000231c38 sp=0xc000231bd8 pc=0x41a56f
runtime.gcAssistAlloc(0xc0004d0ea0)
        /tmp/workdir/go/src/runtime/mgcmark.go:483 +0x1fc fp=0xc000231c98 sp=0xc000231c38 pc=0x41e15c
runtime.mallocgc(0x200, 0x0, 0x0)
        /tmp/workdir/go/src/runtime/malloc.go:913 +0x11a fp=0xc000231d08 sp=0xc000231c98 pc=0x40deda

[bcmills]

In https://storage.googleapis.com/go-build-log/c2fb4188/netbsd-amd64-9_0_06fa9a39.log, the relevant syscalls seem to be:

goroutine 0 [idle]:
runtime.kevent()
	/tmp/workdir/go/src/runtime/sys_netbsd_amd64.s:432 +0x23 fp=0x7f7fcbbff338 sp=0x7f7fcbbff330 pc=0x46ba03
runtime.netpoll(0x9847fe16ad?)
	/tmp/workdir/go/src/runtime/netpoll_kqueue.go:130 +0x10c fp=0x7f7fcbbffdc8 sp=0x7f7fcbbff338 pc=0x432fac
runtime.findRunnable()
	/tmp/workdir/go/src/runtime/proc.go:2952 +0x765 fp=0x7f7fcbbffed0 sp=0x7f7fcbbffdc8 pc=0x43e605
runtime.schedule()
	/tmp/workdir/go/src/runtime/proc.go:3341 +0xb1 fp=0x7f7fcbbfff08 sp=0x7f7fcbbffed0 pc=0x43f791
runtime.mstart1()
	/tmp/workdir/go/src/runtime/proc.go:1499 +0xcd fp=0x7f7fcbbfff30 sp=0x7f7fcbbfff08 pc=0x43bb8d
runtime.mstart0()
	/tmp/workdir/go/src/runtime/proc.go:1449 +0x79 fp=0x7f7fcbbfff60 sp=0x7f7fcbbfff30 pc=0x43ba99
runtime.mstart()
	/tmp/workdir/go/src/runtime/asm_amd64.s:390 +0x5 fp=0x7f7fcbbfff68 sp=0x7f7fcbbfff60 pc=0x467945

That's in a call to __kevent50:
https://cs.opensource.google/go/go/+/master:src/runtime/sys_netbsd_amd64.s;l=430-432;drc=cb6fc99b32cd5ec80e98f0eaec18bcfffba572ca

---

goroutine 32 [syscall]:
runtime.notetsleepg(0x419eb1?, 0x9d80e0?)
	/tmp/workdir/go/src/runtime/lock_sema.go:294 +0x34 fp=0xc000057bf0 sp=0xc000057ba8 pc=0x40cef4
runtime.gcBgMarkStartWorkers()
	/tmp/workdir/go/src/runtime/mgc.go:1192 +0x38 fp=0xc000057c10 sp=0xc000057bf0 pc=0x41b678
runtime.gcStart({0x7f7ff7e995b8?, 0xc00018ec35?, 0x532dd0?})
	/tmp/workdir/go/src/runtime/mgc.go:645 +0x21b fp=0xc000057c98 sp=0xc000057c10 pc=0x41a11b
runtime.mallocgc(0x200, 0x0, 0x0)
	/tmp/workdir/go/src/runtime/malloc.go:1145 +0x7f4 fp=0xc000057d08 sp=0xc000057c98 pc=0x40e5b4

That's acquiring a semaphore, but the stack trace is truncated:
https://cs.opensource.google/go/go/+/master:src/runtime/lock_sema.go;l=294-295;drc=19309779ac5e2f5a2fd3cbb34421dafb2855ac21

It's probably in ___lwp_park60?

• https://cs.opensource.google/go/go/+/master:src/runtime/os_netbsd.go;l=181;drc=5999a28de80cf685b037e53c8dc9a511f11fe7b2
• https://cs.opensource.google/go/go/+/master:src/runtime/sys_netbsd_arm64.s;l=95;drc=cb6fc99b32cd5ec80e98f0eaec18bcfffba572ca

---

goroutine 39 [syscall]:
syscall.Syscall6(0x1e1, 0x1, 0x36e4, 0x0, 0x10020, 0x0, 0x0)
	/tmp/workdir/go/src/syscall/asm_unix_amd64.s:39 +0x5 fp=0xc00005cd28 sp=0xc00005cd20 pc=0x478245
os.(*Process).blockUntilWaitable(0xc0001ca000)
	/tmp/workdir/go/src/os/wait_wait6.go:33 +0x5a fp=0xc00005cd90 sp=0xc00005cd28 pc=0x49d6da
os.(*Process).wait(0xc0001ca000)
	/tmp/workdir/go/src/os/exec_unix.go:22 +0x28 fp=0xc00005cdf0 sp=0xc00005cd90 pc=0x497d08
os.(*Process).Wait(...)
	/tmp/workdir/go/src/os/exec.go:132

goroutine 45 [syscall]:
syscall.Syscall6(0x1e1, 0x1, 0x37ca, 0x0, 0x10020, 0x0, 0x0)
	/tmp/workdir/go/src/syscall/asm_unix_amd64.s:39 +0x5 fp=0xc000205dd8 sp=0xc000205dd0 pc=0x478245
os.(*Process).blockUntilWaitable(0xc00024c090)
	/tmp/workdir/go/src/os/wait_wait6.go:33 +0x5a fp=0xc000205e40 sp=0xc000205dd8 pc=0x49d6da
os.(*Process).wait(0xc00024c090)
	/tmp/workdir/go/src/os/exec_unix.go:22 +0x28 fp=0xc000205ea0 sp=0xc000205e40 pc=0x497d08
os.(*Process).Wait(...)
	/tmp/workdir/go/src/os/exec.go:132

Those are both wait6 with P_PID, a specific PID, WEXITED|WNOWAIT, and null pointers for everything else.

[bcmills]

https://storage.googleapis.com/go-build-log/267326a7/netbsd-amd64-9_0_fc356467.log has these:

            goroutine 0 [idle]:
            runtime.lwp_park()
            	/tmp/workdir/go/src/runtime/sys_netbsd_amd64.s:100 +0x23 fp=0x7f7fcadefd00 sp=0x7f7fcadefcf8 pc=0x46ce83
            runtime.semasleep(0xffffffffffffffff)
            	/tmp/workdir/go/src/runtime/os_netbsd.go:181 +0x125 fp=0x7f7fcadefd60 sp=0x7f7fcadefd00 pc=0x433e85
            runtime.notesleep(0x4460b9?)
            	/tmp/workdir/go/src/runtime/lock_sema.go:181 +0x71 fp=0x7f7fcadefd98 sp=0x7f7fcadefd60 pc=0x40d191
            runtime.mPark(...)
            	/tmp/workdir/go/src/runtime/proc.go:1524
            runtime.stopm()
            	/tmp/workdir/go/src/runtime/proc.go:2315 +0x8c fp=0x7f7fcadefdc8 sp=0x7f7fcadefd98 pc=0x43da4c
            runtime.findRunnable()
            	/tmp/workdir/go/src/runtime/proc.go:2988 +0xabc fp=0x7f7fcadefed0 sp=0x7f7fcadefdc8 pc=0x43f29c

(___lwp_park60)

            goroutine 15 [syscall]:
            runtime.notetsleepg(0x41a531?, 0xfc54a0?)
            	/tmp/workdir/go/src/runtime/lock_sema.go:294 +0x34 fp=0xc000604ce0 sp=0xc000604c98 pc=0x40d574
            runtime.gcBgMarkStartWorkers()
            	/tmp/workdir/go/src/runtime/mgc.go:1192 +0x38 fp=0xc000604d00 sp=0xc000604ce0 pc=0x41bcf8
            runtime.gcStart({0x7f7ff7e9ad28?, 0x15?, 0x604dc0?})
            	/tmp/workdir/go/src/runtime/mgc.go:645 +0x21b fp=0xc000604d88 sp=0xc000604d00 pc=0x41a79b

(also ___lwp_park60, but reached in a different way)

There are no other syscalls in flight that I can see, but I'm not very familiar with runtime debugging (especially on NetBSD).

[bcmills]

I will also note that ___lwp_park60 is the same system call observed in the C program in #50138 (comment). 🙂