runtime: fatal error: checkdead: runnable g

[Mr1X]

What version of Go are you using (go version)?

$ go version
go version go1.20.1 linux/amd64

Does this issue reproduce with the latest release?

only once, run 142d in production

What operating system and processor architecture are you using (go env)?

run application at k8s, docker image from centos7 x86_64 GNU/Linux

go env Output

$ go env

What did you do?

I didn't do anything，it`s an grpc service，normal running 142d, but get a panic today

What did you see instead?

Below is some panic log, if needed，i can provide complete runtime panic output log

runtime: checkdead: find g 18322774 in status 1
fatal error: checkdead: runnable g

runtime stack:
runtime.throw({0x117acd1?, 0x0?})
	/usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc000191d58 sp=0xc000191d28 pc=0x436fdd
runtime.checkdead.func1(0xc0065d3520)
	/usr/local/go/src/runtime/proc.go:5216 +0xe5 fp=0xc000191d90 sp=0xc000191d58 pc=0x444c85
runtime.forEachG(0xc000191df0)
	/usr/local/go/src/runtime/proc.go:591 +0x5c fp=0xc000191dc0 sp=0xc000191d90 pc=0x43a77c
runtime.checkdead()
	/usr/local/go/src/runtime/proc.go:5203 +0xf7 fp=0xc000191e10 sp=0xc000191dc0 pc=0x444977
runtime.mput(0xc000191e40?)
	/usr/local/go/src/runtime/proc.go:5696 +0x33 fp=0xc000191e20 sp=0xc000191e10 pc=0x445ef3
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:2332 +0x67 fp=0xc000191e50 sp=0xc000191e20 pc=0x43dba7
runtime.findRunnable()
	/usr/local/go/src/runtime/proc.go:3007 +0xabc fp=0xc000191f58 sp=0xc000191e50 pc=0x43f41c
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:3360 +0xb1 fp=0xc000191f90 sp=0xc000191f58 pc=0x440251
runtime.park_m(0xc0003264e0?)
	/usr/local/go/src/runtime/proc.go:3511 +0x12d fp=0xc000191fc0 sp=0xc000191f90 pc=0x44076d
runtime.mcall()
	/usr/local/go/src/runtime/asm_amd64.s:452 +0x43 fp=0xc000191fd0 sp=0xc000191fc0 pc=0x469ce3

goroutine 1 [IO wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000193a08 sp=0xc0001939e8 pc=0x439cf6
runtime.netpollblock(0x7f7ff55f7e38?, 0x40482f?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc000193a40 sp=0xc000193a08 pc=0x432657
internal/poll.runtime_pollWait(0x7f7ff5f5a0c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc000193a60 sp=0xc000193a40 pc=0x4665c9
internal/poll.(*pollDesc).wait(0xc0010c7a00?, 0x4?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc000193a88 sp=0xc000193a60 pc=0x4d9e72
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0010c7a00)
	/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd fp=0xc000193b30 sp=0xc000193a88 pc=0x4df77d
net.(*netFD).accept(0xc0010c7a00)
	/usr/local/go/src/net/fd_unix.go:172 +0x35 fp=0xc000193be8 sp=0xc000193b30 pc=0x616635
net.(*TCPListener).accept(0xc0006a60c0)
	/usr/local/go/src/net/tcpsock_posix.go:148 +0x25 fp=0xc000193c10 sp=0xc000193be8 pc=0x6300a5
net.(*TCPListener).Accept(0xc0006a60c0)
	/usr/local/go/src/net/tcpsock.go:297 +0x3d fp=0xc000193c40 sp=0xc000193c10 pc=0x62f19d
google.golang.org/grpc.(*Server).Serve(0xc000342000, {0x12d3000, 0xc0006a60c0})
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362 fp=0xc000193d58 sp=0xc000193c40 pc=0x8e77c2
internal/pkg/middleware/grpc.(*GrpcServer).Serve(0xc0008bae60)
	/middleware/grpc/grpc_server.go:97 +0xc5 fp=0xc000193db0 sp=0xc000193d58 pc=0x94bee5
main.main()
	/app/main.go:94 +0x4a9 fp=0xc000193f80 sp=0xc000193db0 pc=0xf091a9
runtime.main()
	/usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc000193fe0 sp=0xc000193f80 pc=0x4398c7
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000193fe8 sp=0xc000193fe0 pc=0x46bde1

goroutine 2 [force gc (idle), 239 minutes]:
runtime.gopark(0x14d316204e4f521?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e8fb0 sp=0xc0000e8f90 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc0000e8fe0 sp=0xc0000e8fb0 pc=0x439b30
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e8fe8 sp=0xc0000e8fe0 pc=0x46bde1
created by runtime.init.6
	/usr/local/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e9780 sp=0xc0000e9760 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
	/usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0000e97c8 sp=0xc0000e9780 pc=0x42437e
runtime.gcenable.func1()
	/usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000e97e0 sp=0xc0000e97c8 pc=0x419606
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e97e8 sp=0xc0000e97e0 pc=0x46bde1
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:178 +0x6b

goroutine 4 [runnable]:
runtime.gopark(0x14d3e54d28b3b6e?, 0x105aabe?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e9f70 sp=0xc0000e9f50 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x1b42920)
	/usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc0000e9fa0 sp=0xc0000e9f70 pc=0x422273
runtime.bgscavenge(0x0?)
	/usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc0000e9fc8 sp=0xc0000e9fa0 pc=0x422865
runtime.gcenable.func2()
	/usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc0000e9fe0 sp=0xc0000e9fc8 pc=0x4195a6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e9fe8 sp=0xc0000e9fe0 pc=0x46bde1
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 17 [finalizer wait]:
runtime.gopark(0x0?, 0x11ce3d8?, 0x20?, 0x80?, 0x2000000020?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e8628 sp=0xc0000e8608 pc=0x439cf6
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc0000e87e0 sp=0xc0000e8628 pc=0x418647
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e87e8 sp=0xc0000e87e0 pc=0x46bde1
created by runtime.createfing
	/usr/local/go/src/runtime/mfinal.go:163 +0x45

goroutine 22 [chan receive, 205514 minutes]:
runtime.gopark(0x4064f1?, 0x1017420?, 0x90?, 0x47?, 0xc00030e6b0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e46a8 sp=0xc0000e4688 pc=0x439cf6
runtime.chanrecv(0xc0001b27e0, 0xc0000e4788, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc0000e4738 sp=0xc0000e46a8 pc=0x406f9d
runtime.chanrecv2(0x405d3d?, 0xc0000e47a8?)
	/usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc0000e4760 sp=0xc0000e4738 pc=0x406ad8
github.com/alecthomas/log4go.(*ConsoleLogWriter).run(0xc0001aeea0, {0x12ce360, 0xc0001b0008})
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:33 +0xa5 fp=0xc0000e47b8 sp=0xc0000e4760 pc=0xe3bb65
github.com/alecthomas/log4go.NewConsoleLogWriter.func1()
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:26 +0x2e fp=0xc0000e47e0 sp=0xc0000e47b8 pc=0xe3ba2e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e47e8 sp=0xc0000e47e0 pc=0x46bde1
created by github.com/alecthomas/log4go.NewConsoleLogWriter
	/gobuild/go/pkg/mod/github.com/alecthomas/log4go@v0.0.0-20180109082532-d146e6b86faa/termlog.go:26 +0xf5

goroutine 8 [select, 205514 minutes]:
runtime.gopark(0xc0000ea788?, 0x2?, 0x88?, 0xda?, 0xc0000ea784?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000ea610 sp=0xc0000ea5f0 pc=0x439cf6
runtime.selectgo(0xc0000ea788, 0xc0000ea780, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0000ea750 sp=0xc0000ea610 pc=0x449e9e
database/sql.(*DB).connectionOpener(0xc0002ed2b0, {0x12d39a0, 0xc000116190})
	/usr/local/go/src/database/sql/sql.go:1218 +0x8d fp=0xc0000ea7b8 sp=0xc0000ea750 pc=0x78300d
database/sql.OpenDB.func1()
	/usr/local/go/src/database/sql/sql.go:791 +0x2e fp=0xc0000ea7e0 sp=0xc0000ea7b8 pc=0x78138e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000ea7e8 sp=0xc0000ea7e0 pc=0x46bde1
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:791 +0x18d

goroutine 14 [select, 205514 minutes]:
runtime.gopark(0xc0000ebf80?, 0x2?, 0x0?, 0x0?, 0xc0000ebf40?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000ebdc0 sp=0xc0000ebda0 pc=0x439cf6
runtime.selectgo(0xc0000ebf80, 0xc0000ebf3c, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0000ebf00 sp=0xc0000ebdc0 pc=0x449e9e
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/gobuild/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:614 +0xaa fp=0xc0000ebfe0 sp=0xc0000ebf00 pc=0x95d06a
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000ebfe8 sp=0xc0000ebfe0 pc=0x46bde1
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/gobuild/go/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/connection.go:611 +0x10a

[mknyszek]

In triage: @prattmic thinks this could be a race with a goroutine exiting, since that goroutine doesn't appear in the stack trace. However, unless this is reproducible, I'm not sure this is enough information to come to a conclusion.

[mknyszek]

If you see it again, please report back! Thanks.

[Mr1X]

This is complete panic stack trace, goroutine 18322774 is in log file. I hope it can help you analyze
panic.log

[prattmic]

Ah, thanks. For reference, there are two runnable goroutines in the full dump:

goroutine 4 [runnable]:
runtime.gopark(0x14d3e54d28b3b6e?, 0x105aabe?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000e9f70 sp=0xc0000e9f50 pc=0x439cf6
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x1b42920)
	/usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc0000e9fa0 sp=0xc0000e9f70 pc=0x422273
runtime.bgscavenge(0x0?)
	/usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc0000e9fc8 sp=0xc0000e9fa0 pc=0x422865
runtime.gcenable.func2()
	/usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc0000e9fe0 sp=0xc0000e9fc8 pc=0x4195a6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000e9fe8 sp=0xc0000e9fe0 pc=0x46bde1
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 18322774 [runnable]:
runtime.gopark(0xc000f04a78?, 0xb?, 0x0?, 0x0?, 0x20?)
	/usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0006cfab8 sp=0xc0006cfa98 pc=0x439cf6
runtime.netpollblock(0x47c5c5?, 0x40482f?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0006cfaf0 sp=0xc0006cfab8 pc=0x432657
internal/poll.runtime_pollWait(0x7f7ff56775c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0006cfb10 sp=0xc0006cfaf0 pc=0x4665c9
internal/poll.(*pollDesc).wait(0xc001069a00?, 0xc0005d6000?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0006cfb38 sp=0xc0006cfb10 pc=0x4d9e72
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001069a00, {0xc0005d6000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc0006cfbd0 sp=0xc0006cfb38 pc=0x4db259
net.(*netFD).Read(0xc001069a00, {0xc0005d6000?, 0x18?, 0x1066660?})
	/usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc0006cfc18 sp=0xc0006cfbd0 pc=0x6144a9
net.(*conn).Read(0xc000756628, {0xc0005d6000?, 0xc0006cfcd0?, 0x44f056?})
	/usr/local/go/src/net/net.go:183 +0x45 fp=0xc0006cfc60 sp=0xc0006cfc18 pc=0x626c25
net.(*TCPConn).Read(0x1010000000001?, {0xc0005d6000?, 0x1?, 0x0?})
	<autogenerated>:1 +0x29 fp=0xc0006cfc90 sp=0xc0006cfc60 pc=0x638fc9
bufio.(*Reader).Read(0xc001559c20, {0xc0007c5618, 0x9, 0x0?})
	/usr/local/go/src/bufio/bufio.go:237 +0x1bb fp=0xc0006cfcc8 sp=0xc0006cfc90 pc=0x50b1bb
io.ReadAtLeast({0x12cca20, 0xc001559c20}, {0xc0007c5618, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:332 +0x9a fp=0xc0006cfd10 sp=0xc0006cfcc8 pc=0x4d453a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0007c5618?, 0x9?, 0x0?}, {0x12cca20?, 0xc001559c20?})
	/gobuild/go/pkg/mod/golang.org/x/net@v0.0.0-20210226172049-e18ecbb05110/http2/frame.go:237 +0x6e fp=0xc0006cfd60 sp=0xc0006cfd10 pc=0x8528ce
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007c55e0)
	/gobuild/go/pkg/mod/golang.org/x/net@v0.0.0-20210226172049-e18ecbb05110/http2/frame.go:492 +0x95 fp=0xc0006cfe08 sp=0xc0006cfd60 pc=0x853055
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000951b00, 0x0?, 0x0?)
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0x174 fp=0xc0006cff18 sp=0xc0006cfe08 pc=0x87e354
google.golang.org/grpc.(*Server).serveStreams(0xc000342000, {0x12d7ff8?, 0xc000951b00})
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142 fp=0xc0006cff80 sp=0xc0006cff18 pc=0x8e88c2
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46 fp=0xc0006cffe0 sp=0xc0006cff80 pc=0x8e8186
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0006cffe8 sp=0xc0006cffe0 pc=0x46bde1
created by google.golang.org/grpc.(*Server).handleRawConn
	/gobuild/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185