Go version
go1.26.2 windows/amd64
Output of go env in your module/workspace:
set AR=ar
set CC=gcc
set CGO_CFLAGS=-O2 -g
set CGO_CPPFLAGS=
set CGO_CXXFLAGS=-O2 -g
set CGO_ENABLED=1
set CGO_FFLAGS=-O2 -g
set CGO_LDFLAGS=-O2 -g
set CXX=g++
set GCCGO=gccgo
set GO111MODULE=
set GOAMD64=v1
set GOARCH=amd64
set GOAUTH=netrc
set GOBIN=C:\Users\Administrator\go\bin
set GOCACHE=C:\Users\Administrator\AppData\Local\go-build
set GOCACHEPROG=
set GODEBUG=
set GOENV=C:\Users\Administrator\AppData\Roaming\go\env
set GOEXE=.exe
set GOEXPERIMENT=
set GOFIPS140=off
set GOFLAGS=
set GOGCCFLAGS=-m64 -mthreads -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=C:\Users\ADMINI~1\AppData\Local\Temp\go-build3081638714=/tmp/go-build -gno-record-gcc-switches
set GOHOSTARCH=amd64
set GOHOSTOS=windows
set GOINSECURE=
set GOMOD=NUL
set GOMODCACHE=C:\Users\Administrator\go\pkg\mod
set GONOPROXY=github.com/NozomiNetworks
set GONOSUMDB=github.com/NozomiNetworks
set GOOS=windows
set GOPATH=C:\Users\Administrator\go
set GOPRIVATE=github.com/NozomiNetworks
set GOPROXY=https://proxy.golang.org,direct
set GOROOT=C:\go
set GOSUMDB=sum.golang.org
set GOTELEMETRY=local
set GOTELEMETRYDIR=C:\Users\Administrator\AppData\Roaming\go\telemetry
set GOTMPDIR=
set GOTOOLCHAIN=auto
set GOTOOLDIR=C:\go\pkg\tool\windows_amd64
set GOVCS=
set GOVERSION=go1.26.2
set GOWORK=
set PKG_CONFIG=pkg-config
What did you do?
our CI tests started to have this sporadic panic since we upgraded from Go 1.25.8 to Go 1.26.2
The panic occurs only on windows and during execution of tests, we didn't notice any failure in the built executable.
The failure can occur in different code paths in a suite of hundred of tests, since the error is very sporadic, we are not able to isolate it and provide a reproducible example.
But we noticed that the error always occur during a windows systemcall, in our tests we run goleak so we supposed a possible incompatibility with Go 1.26 runtime or bug in the Go runtime itself.
What did you see happen?
Here the stacktrace:
runtime: waitforsingleobject wait_failed; errno=5
fatal error: runtime: setevent failed; errno=6
fatal error: runtime.semawakeup
runtime.semasleep wait_failed
runtime stack:
runtime.throw({0x7ff66dd39cdb?, 0x1cc72fc2b008?})
C:/go/src/runtime/panic.go:1229 +0x4d fp=0x7f7efff618 sp=0x7f7efff5e8 pc=0x7ff66d8a68ed
runtime.semawakeup.func1()
C:/go/src/runtime/os_windows.go:697 +0x6a fp=0x7f7efff640 sp=0x7f7efff618 pc=0x7ff66d8a138a
runtime.semawakeup(0x3ff86241b2d78?)
C:/go/src/runtime/os_windows.go:695 +0x4b fp=0x7f7efff678 sp=0x7f7efff640 pc=0x7ff66d86a90b
runtime.notewakeup(0x7ff66d9a619b?)
C:/go/src/runtime/lock_sema.go:42 +0x45 fp=0x7f7efff698 sp=0x7f7efff678 pc=0x7ff66d83cac5
runtime.exitsyscallTryGetP(0x1cc72fc14008?)
C:/go/src/runtime/proc.go:5050 +0x7e fp=0x7f7efff6e8 sp=0x7f7efff698 pc=0x7ff66d87aafe
runtime.exitsyscall.func3()
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1cc72fdb3fe8 sp=0x1cc72fdb3fe0 pc=0x7ff66d8ae4c1
created by runtime.gcBgMarkStartWorkers in goroutine 14
C:/go/src/runtime/mgc.go:1695 +0x105
goroutine 16 gp=0x1cc72fc212c0 m=nil [GC worker (idle)]:
runtime.gopark(0x3ff8622250c28?, 0x0?, 0x0?, 0x0?, 0x0?)
C:/go/src/runtime/proc.go:462 +0xce fp=0x1cc72fc35f40 sp=0x1cc72fc35f20 pc=0x7ff66d8a6a0e
runtime.gcBgMarkWorker(0x1cc7300e2690)
C:/go/src/runtime/mgc.go:1791 +0xeb fp=0x1cc72fc35fc8 sp=0x1cc72fc35f40 pc=0x7ff66d84af8b
runtime.gcBgMarkStartWorkers.gowrap1()
C:/go/src/runtime/mgc.go:1695 +0x17 fp=0x1cc72fc35fe0 sp=0x1cc72fc35fc8 pc=0x7ff66d84ae77
runtime.goexit({})
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1cc72fc35fe8 sp=0x1cc72fc35fe0 pc=0x7ff66d8ae4c1
created by runtime.gcBgMarkStartWorkers in goroutine 14
C:/go/src/runtime/mgc.go:1695 +0x105
goroutine 50 gp=0x1cc72fc214a0 m=nil [GC worker (idle)]:
runtime.gopark(0x3ff8622aa3ab0?, 0x3?, 0x0?, 0x0?, 0x0?)
C:/go/src/runtime/proc.go:462 +0xce fp=0x1cc7301d7f40 sp=0x1cc7301d7f20 pc=0x7ff66d8a6a0e
runtime.gcBgMarkWorker(0x1cc7300e2690)
C:/go/src/runtime/mgc.go:1791 +0xeb fp=0x1cc7301d7fc8 sp=0x1cc7301d7f40 pc=0x7ff66d84af8b
runtime.gcBgMarkStartWorkers.gowrap1()
C:/go/src/runtime/mgc.go:1695 +0x17 fp=0x1cc7301d7fe0 sp=0x1cc7301d7fc8 pc=0x7ff66d84ae77
runtime.goexit({})
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1cc7301d7fe8 sp=0x1cc7301d7fe0 pc=0x7ff66d8ae4c1
created by runtime.gcBgMarkStartWorkers in goroutine 14
C:/go/src/runtime/mgc.go:1695 +0x105
goroutine 51 gp=0x1cc72fc21680 m=nil [GC worker (idle)]:
runtime.gopark(0x3ff8622aa3ab0?, 0x0?, 0x0?, 0x0?, 0x0?)
C:/go/src/runtime/proc.go:462 +0xce fp=0x1cc7301d9f40 sp=0x1cc7301d9f20 pc=0x7ff66d8a6a0e
runtime.gcBgMarkWorker(0x1cc7300e2690)
C:/go/src/runtime/mgc.go:1791 +0xeb fp=0x1cc7301d9fc8 sp=0x1cc7301d9f40 pc=0x7ff66d84af8b
runtime.gcBgMarkStartWorkers.gowrap1()
C:/go/src/runtime/mgc.go:1695 +0x17 fp=0x1cc7301d9fe0 sp=0x1cc7301d9fc8 pc=0x7ff66d84ae77
runtime.goexit({})
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1cc7301d9fe8 sp=0x1cc7301d9fe0 pc=0x7ff66d8ae4c1
created by runtime.gcBgMarkStartWorkers in goroutine 14
C:/go/src/runtime/mgc.go:1695 +0x105
runtime: setevent failed; errno=6
fatal error: runtime.semawakeup
panic during panic
runtime stack:
runtime.throw({0x7ff66dd39cdb?, 0x1cc72fc2b008?})
C:/go/src/runtime/panic.go:1229 +0x4d fp=0x7f7efff418 sp=0x7f7efff3e8 pc=0x7ff66d8a68ed
runtime.semawakeup.func1()
C:/go/src/runtime/os_windows.go:697 +0x6a fp=0x7f7efff440 sp=0x7f7efff418 pc=0x7ff66d8a138a
runtime.semawakeup(0x7ff66d897a25?)
C:/go/src/runtime/os_windows.go:695 +0x4b fp=0x7f7efff478 sp=0x7f7efff440 pc=0x7ff66d86a90b
runtime.unlock2Wake(0x7ff66d897a60?, 0x60?, 0x1cc72fc88960?)
C:/go/src/runtime/lock_spinbit.go:461 +0x1bc fp=0x7f7efff4b8 sp=0x7f7efff478 pc=0x7ff66d83d95c
runtime.unlock2(0x1cc72fc20960?)
C:/go/src/runtime/lock_spinbit.go:314 +0xb3 fp=0x7f7efff4f0 sp=0x7f7efff4b8 pc=0x7ff66d83d673
runtime.unlockWithRank(...)
C:/go/src/runtime/lockrank_off.go:35
runtime.unlock(...)
C:/go/src/runtime/lock_spinbit.go:261
runtime.dopanic_m(0x1cc72fc20960, 0x7ff66d8a68ed, 0x7f7efff5e8, 0x0)
C:/go/src/runtime/panic.go:1604 +0x2a8 fp=0x7f7efff560 sp=0x7f7efff4f0 pc=0x7ff66d86e668
runtime.fatalthrow.func1()
C:/go/src/runtime/panic.go:1438 +0x45 fp=0x7f7efff5a8 sp=0x7f7efff560 pc=0x7ff66d86e0a5
runtime.fatalthrow(0x7efff5f0?)
C:/go/src/runtime/panic.go:1431 +0x58 fp=0x7f7efff5e8 sp=0x7f7efff5a8 pc=0x7ff66d86e038
runtime.throw({0x7ff66dd39cdb?, 0x1cc72fc2b008?})
C:/go/src/runtime/panic.go:1229 +0x4d fp=0x7f7efff618 sp=0x7f7efff5e8 pc=0x7ff66d8a68ed
runtime.semawakeup.func1()
C:/go/src/runtime/os_windows.go:697 +0x6a fp=0x7f7efff640 sp=0x7f7efff618 pc=0x7ff66d8a138a
runtime.semawakeup(0x3ff86241b2d78?)
C:/go/src/runtime/os_windows.go:695 +0x4b fp=0x7f7efff678 sp=0x7f7efff640 pc=0x7ff66d86a90b
runtime.notewakeup(0x7ff66d9a619b?)
C:/go/src/runtime/lock_sema.go:42 +0x45 fp=0x7f7efff698 sp=0x7f7efff678 pc=0x7ff66d83cac5
runtime.exitsyscallTryGetP(0x1cc72fc14008?)
C:/go/src/runtime/proc.go:5050 +0x7e fp=0x7f7efff6e8 sp=0x7f7efff698 pc=0x7ff66d87aafe
runtime.exitsyscall.func3()
C:/go/src/runtime/proc.go:4961 +0x1b fp=0x7f7efff720 sp=0x7f7efff6e8 pc=0x7ff66d87a89b
runtime.systemstack(0x1cc72fc20960)
C:/go/src/runtime/asm_amd64.s:562 +0x49 fp=0x7f7efff730 sp=0x7f7efff720 pc=0x7ff66d8ac889
and another one:
runtime: waitforsingleobject wait_failed; errno=6
fatal error: runtime.semasleep wait_failed
runtime stack:
runtime.throw({0x7ff67ab5ac9b?, 0x1766b6407008?})
C:/go/src/runtime/panic.go:1229 +0x4d fp=0xa067fffc18 sp=0xa067fffbe8 pc=0x7ff67a6c694d
runtime.semasleep.func2()
C:/go/src/runtime/os_windows.go:750 +0x6a fp=0xa067fffc40 sp=0xa067fffc18 pc=0x7ff67a6c14aa
runtime.semasleep(0xdf8475800?)
C:/go/src/runtime/os_windows.go:748 +0x1f6 fp=0xa067fffcd0 sp=0xa067fffc40 pc=0x7ff67a68abf6
runtime.notetsleep_internal(0xa067fffd30?, 0xdf8475800, 0x110?, 0x1766b64485a0?)
C:/go/src/runtime/lock_sema.go:113 +0xe6 fp=0xa067fffd10 sp=0xa067fffcd0 pc=0x7ff67a65cd26
runtime.notetsleep(0x7ff67af8b4a0, 0xdf8475800)
C:/go/src/runtime/lock_sema.go:164 +0x3f fp=0xa067fffd40 sp=0xa067fffd10 pc=0x7ff67a65ceff
runtime.sysmon()
C:/go/src/runtime/proc.go:6542 +0x225 fp=0xa067fffdd8 sp=0xa067fffd40 pc=0x7ff67a69e925
runtime.mstart1()
C:/go/src/runtime/proc.go:1935 +0x9d fp=0xa067fffe00 sp=0xa067fffdd8 pc=0x7ff67a694afd
runtime.mstart0()
C:/go/src/runtime/proc.go:1888 +0x6a fp=0xa067fffe28 sp=0xa067fffe00 pc=0x7ff67a694a4a
runtime.mstart()
C:/go/src/runtime/asm_amd64.s:426 +0xc fp=0xa067fffe30 sp=0xa067fffe28 pc=0x7ff67a6cc50c
goroutine 1 gp=0x1766b6448000 m=nil [chan receive]:
runtime.gopark(0x7ff67a6cdebf?, 0x30066b651b8d0?, 0x8?, 0x1e?, 0x20b0b301b90?)
C:/go/src/runtime/proc.go:462 +0xce fp=0x1766b6431888 sp=0x1766b6431868 pc=0x7ff67a6c6a6e
runtime.chanrecv(0x1766b64bb400, 0x1766b651b96f, 0x1)
C:/go/src/runtime/chan.go:667 +0x4ae fp=0x1766b6431900 sp=0x1766b6431888 pc=0x7ff67a656c0e
runtime.chanrecv1(0x7ff67af87a40?, 0x7ff67ab95040?)
C:/go/src/runtime/chan.go:509 +0x12 fp=0x1766b6431928 sp=0x1766b6431900 pc=0x7ff67a656732
testing.(*T).Run(0x1766b650a400, {0x7ff67ab579ef?, 0x1766b651ba18?}, 0x7ff67ab793d8)
C:/go/src/testing/testing.go:2109 +0x4c7 fp=0x1766b64319f0 sp=0x1766b6431928 pc=0x7ff67a7c3da7
testing.runTests.func1(0x1766b650a400)
C:/go/src/testing/testing.go:2585 +0x37 fp=0x1766b6431a30 sp=0x1766b64319f0 pc=0x7ff67a7c6197
testing.tRunner(0x1766b650a400, 0x1766b651bb38)
C:/go/src/testing/testing.go:2036 +0xc3 fp=0x1766b6431a80 sp=0x1766b6431a30 pc=0x7ff67a7c2de3
testing.runTests({0x7ff67ab5a7d9, 0x1d}, {0x7ff67ab67636, 0x2f}, 0x1766b6494300, {0x7ff67af80640, 0x1c, 0x1c}, {0xc27200ff5949b1a0, 0x1176606edc5, ...})
C:/go/src/testing/testing.go:2583 +0x4f0 fp=0x1766b6431b68 sp=0x1766b6431a80 pc=0x7ff67a7c6050
testing.(*M).Run(0x1766b65c2320)
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1766b69fdfe8 sp=0x1766b69fdfe0 pc=0x7ff67a6ce261
created by runtime.gcBgMarkStartWorkers in goroutine 45
C:/go/src/runtime/mgc.go:1695 +0x105
goroutine 49 gp=0x1766b64890e0 m=nil [GC worker (idle)]:
runtime.gopark(0x194213b7154958?, 0x0?, 0x0?, 0x0?, 0x0?)
C:/go/src/runtime/proc.go:462 +0xce fp=0x1766b69fff40 sp=0x1766b69fff20 pc=0x7ff67a6c6a6e
runtime.gcBgMarkWorker(0x1766b6941f10)
C:/go/src/runtime/mgc.go:1791 +0xeb fp=0x1766b69fffc8 sp=0x1766b69fff40 pc=0x7ff67a66af8b
runtime.gcBgMarkStartWorkers.gowrap1()
C:/go/src/runtime/mgc.go:1695 +0x17 fp=0x1766b69fffe0 sp=0x1766b69fffc8 pc=0x7ff67a66ae77
runtime.goexit({})
C:/go/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1766b69fffe8 sp=0x1766b69fffe0 pc=0x7ff67a6ce261
created by runtime.gcBgMarkStartWorkers in goroutine 45
C:/go/src/runtime/mgc.go:1695 +0x105
What did you expect to see?
No panic
Go version
go1.26.2 windows/amd64
Output of
go envin your module/workspace:What did you do?
our CI tests started to have this sporadic panic since we upgraded from Go 1.25.8 to Go 1.26.2
The panic occurs only on windows and during execution of tests, we didn't notice any failure in the built executable.
The failure can occur in different code paths in a suite of hundred of tests, since the error is very sporadic, we are not able to isolate it and provide a reproducible example.
But we noticed that the error always occur during a windows systemcall, in our tests we run goleak so we supposed a possible incompatibility with Go 1.26 runtime or bug in the Go runtime itself.
What did you see happen?
Here the stacktrace:
and another one:
What did you expect to see?
No panic