goroutine 920 [running]: runtime/pprof.writeGoroutineStacks({0x6548840, 0xc48830f610}) /usr/local/go/src/runtime/pprof/pprof.go:703 +0x6a runtime/pprof.writeGoroutine({0x6548840?, 0xc48830f610?}, 0x5d6a158?) /usr/local/go/src/runtime/pprof/pprof.go:692 +0x25 runtime/pprof.(*Profile).WriteTo(0x5b7e6db?, {0x6548840?, 0xc48830f610?}, 0xc001c13850?) /usr/local/go/src/runtime/pprof/pprof.go:329 +0x146 github.com/pingcap/tidb/util/memoryusagealarm.write({{0x5b7e6db?, 0x0?}, 0x54aa3c0?}, {0xc6a77c2940?, 0xc7555487e0?}) /go/src/github.com/pingcap/tidb/util/memoryusagealarm/memoryusagealarm.go:366 +0x310 github.com/pingcap/tidb/util/memoryusagealarm.(*memoryUsageAlarm).recordProfile(0xc6a77c2940, {0xc6a77c2940, 0x3c}) /go/src/github.com/pingcap/tidb/util/memoryusagealarm/memoryusagealarm.go:351 +0xbd github.com/pingcap/tidb/util/memoryusagealarm.(*memoryUsageAlarm).doRecord(0xc620091f20, 0x7fff04000?, 0x0?, {0x659fa08, 0xc000a66c80}, 0x1) /go/src/github.com/pingcap/tidb/util/memoryusagealarm/memoryusagealarm.go:223 +0xc95 github.com/pingcap/tidb/util/memoryusagealarm.(*memoryUsageAlarm).alarm4ExcessiveMemUsage(0xc620091f20, {0x659fa08, 0xc000a66c80}) /go/src/github.com/pingcap/tidb/util/memoryusagealarm/memoryusagealarm.go:161 +0x316 github.com/pingcap/tidb/util/memoryusagealarm.(*Handle).Run(0xc0011dd0e0) /go/src/github.com/pingcap/tidb/util/memoryusagealarm/memoryusagealarm.go:66 +0x10c created by main.createServer in goroutine 1 /go/src/github.com/pingcap/tidb/tidb-server/main.go:1188 +0x267 goroutine 1 [chan receive, 3774 minutes]: github.com/pingcap/tidb/server.(*Server).Run(0xc000a66c80) /go/src/github.com/pingcap/tidb/server/server.go:447 +0x1df main.main() /go/src/github.com/pingcap/tidb/tidb-server/main.go:431 +0x1ca5 goroutine 8 [select, 2 minutes]: github.com/golang/glog.(*fileSink).flushDaemon(0x94f6b98) /go/cache/github.com/golang/glog@v1.1.2/glog_file.go:351 +0xb9 created by github.com/golang/glog.init.1 in goroutine 1 /go/cache/github.com/golang/glog@v1.1.2/glog_file.go:166 +0x135 goroutine 9 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000b67800) /go/cache/go.opencensus.io@v0.23.1-0.20220331163232-052120675fac/stats/view/worker.go:292 +0x9f created by go.opencensus.io/stats/view.init.0 in goroutine 1 /go/cache/go.opencensus.io@v0.23.1-0.20220331163232-052120675fac/stats/view/worker.go:34 +0x8d goroutine 10 [select, 3774 minutes]: github.com/lestrrat-go/httprc.runFetchWorker({0x657b8a8, 0x9529b80}, 0xc0000df6e0, {0x0, 0x0}) /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:135 +0xc9 created by github.com/lestrrat-go/httprc.NewFetcher in goroutine 1 /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:75 +0x1c5 goroutine 11 [select, 3774 minutes]: github.com/lestrrat-go/httprc.runFetchWorker({0x657b8a8, 0x9529b80}, 0xc0000df6e0, {0x0, 0x0}) /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:135 +0xc9 created by github.com/lestrrat-go/httprc.NewFetcher in goroutine 1 /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:75 +0x1c5 goroutine 12 [select, 3774 minutes]: github.com/lestrrat-go/httprc.runFetchWorker({0x657b8a8, 0x9529b80}, 0xc0000df6e0, {0x0, 0x0}) /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:135 +0xc9 created by github.com/lestrrat-go/httprc.NewFetcher in goroutine 1 /go/cache/github.com/lestrrat-go/httprc@v1.0.4/fetcher.go:75 +0x1c5 goroutine 448 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce66e0, {0x657b950, 0xc001c784b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 216 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000de0190, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b761c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 451 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 215 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0479a88, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000f2c000?, 0xc000e34000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000f2c000, {0xc000e34000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000f2c000, {0xc000e34000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd2038, {0xc000e34000?, 0x0?, 0xc001440800?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32000, {0xc000e44040, 0x9, 0x7f8b191acd28?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32000}, {0xc000e44040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000e44040, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e32000?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e44000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000e72000, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 451 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 478 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 499 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 199 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000f300f0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b82000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 459 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 449 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce67c0, {0x657b950, 0xc001c78730}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 189 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0479d70, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001d1a380?, 0xc0019ea000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001d1a380, {0xc0019ea000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001d1a380, {0xc0019ea000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0000ac510, {0xc0019ea000?, 0x0?, 0xc35ca21ecc?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0019c6420, {0xc0019ca120, 0x9, 0x7f8b191acd28?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0019c6420}, {0xc0019ca120, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0019ca120, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc0019c6420?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019ca0e0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0019fa000, 0xc001c0001c?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 455 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 572 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7900, {0x657b950, 0xc001216dc0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 197 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000f64000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 459 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 188 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0019fa000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 455 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 453 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6ae0, {0x657b950, 0xc001c78dc0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 444 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce63a0, {0x657b950, 0xc0000ddb80}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 568 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001216410, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b21c70) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 546 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 450 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6820, {0x657b950, 0xc001c78780}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 432 [chan receive, 39 minutes]: gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun(0xc001c4c0c0) /go/cache/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:379 +0x3f created by gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1 in goroutine 1 /go/cache/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:390 +0x96 goroutine 222 [select, 2 minutes]: github.com/tikv/pd/client.(*tsoClient).tsoDispatcherCheckLoop(0xc001935a00) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:240 +0x176 created by github.com/tikv/pd/client.(*tsoClient).Setup in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_client.go:125 +0x85 goroutine 452 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6a00, {0x657b950, 0xc001c78b40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 433 [select]: github.com/pingcap/tidb/util/cpuprofile.(*parallelCPUProfiler).profilingLoop(0x94f6660) /go/src/github.com/pingcap/tidb/util/cpuprofile/cpuprofile.go:164 +0xf4 github.com/pingcap/tidb/util.WithRecovery(0x1e65687?, 0xc000a8001c?) /go/src/github.com/pingcap/tidb/util/misc.go:96 +0x48 created by github.com/pingcap/tidb/util/cpuprofile.(*parallelCPUProfiler).start in goroutine 1 /go/src/github.com/pingcap/tidb/util/cpuprofile/cpuprofile.go:111 +0x176 goroutine 434 [chan receive]: github.com/pingcap/tidb/util/systimemon.StartMonitor(0x5d69aa8, 0x5d69a98) /go/src/github.com/pingcap/tidb/util/systimemon/systime_mon.go:31 +0x17b created by main.setupMetrics in goroutine 1 /go/src/github.com/pingcap/tidb/tidb-server/main.go:1201 +0x9f goroutine 198 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0479e68, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001c74c80?, 0xc000f4c000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001c74c80, {0xc000f4c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001c74c80, {0xc000f4c000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e97bc8, {0xc000f4c000?, 0x0?, 0x4?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000f2e060, {0xc000f34040, 0x9, 0x7f8b191ac878?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000f2e060}, {0xc000f34040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000f34040, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000f2e060?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f34000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f64000, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 459 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 219 [runnable]: syscall.Syscall(0x1d2d57f?, 0xc00130bb50?, 0x1d2d867?, 0x7ffff800000?) /usr/local/go/src/syscall/syscall_linux.go:69 +0x25 syscall.read(0xc000d4c500?, {0xc000e76000?, 0x0?, 0xc00130bbb0?}) /usr/local/go/src/syscall/zsyscall_linux_amd64.go:721 +0x38 syscall.Read(...) /usr/local/go/src/syscall/syscall_unix.go:181 internal/poll.ignoringEINTRIO(...) /usr/local/go/src/internal/poll/fd_unix.go:736 internal/poll.(*FD).Read(0xc000d4c500, {0xc000e76000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:160 +0x2ae net.(*netFD).Read(0xc000d4c500, {0xc000e76000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd2050, {0xc000e76000?, 0x0?, 0xc2e4e7becc?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32120, {0xc000e44120, 0x9, 0x7f8b191ad1d8?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32120}, {0xc000e44120, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000e44120, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e32120?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e440e0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000e72240, 0x1?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 488 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 148779343 [select, 411 minutes]: github.com/pingcap/tidb/disttask/framework/dispatcher.(*Manager).gcSubtaskHistoryTableLoop(0xc6509fc750) /go/src/github.com/pingcap/tidb/disttask/framework/dispatcher/dispatcher_manager.go:247 +0x10d github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 571 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7860, {0x657b950, 0xc001216be0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 436 [select]: github.com/pingcap/tidb/resourcemanager.(*ResourceManager).Start.func1() /go/src/github.com/pingcap/tidb/resourcemanager/rm.go:63 +0xbb github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 756 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000f65b00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 779 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 621 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001e82af0, {0xc001ea0460, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001e82af0, {0xc001ea0460?, 0xc0016f3ce0?, 0xc001d0b9d8?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001ea0420, {0xc001ea0460?, 0xc001d0ba50?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc001ea0420}, {0xc001ea0460, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c7d9e0, {0xc001ea0460, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc001ea0450, 0x7fffffff) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x7f8a457bf1d0?, 0xc001c7d9e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc001ea0450, {0x7f8ad0162008, 0x9529b80}, 0x1c43665?, {0x0, 0x0}, {0x598e240, 0xc59f868640}, 0x59b3040?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0016ec5b0, {0x598e240?, 0xc59f868640}, 0x1c332da?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x295e125?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001c7d7a0, 0xc001d0be78, 0xc001d0be68) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c7d7a0, {0x598e240?, 0xc59f868640?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0016df0b0) /go/cache/go.etcd.io/etcd/api/v3@v3.5.10/etcdserverpb/rpc.pb.go:6853 +0x46 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc000dc8960) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:458 +0x258 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:301 +0x58 goroutine 649 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 214 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000e72000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 451 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 473 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001aa6840, {0x657b950, 0xc000c865f0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 592 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001003200) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 576 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 221 [select]: github.com/tikv/pd/client.(*tsoBatchController).fetchPendingRequests(0xc000de06e0, {0x657b950, 0xc000de0690}, 0x0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_batch_controller.go:48 +0xc5 github.com/tikv/pd/client.(*tsoClient).handleDispatcher(0xc001935a00, {0x657b950?, 0xc000de0690}, {0x5b7488a, 0x6}, 0xc000de06e0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:422 +0x3df created by github.com/tikv/pd/client.(*tsoClient).createTSODispatcher in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:334 +0x274 goroutine 223 [select, 2 minutes]: github.com/tikv/pd/client.(*tsoClient).tsCancelLoop(0xc001935a00) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:184 +0x1c8 created by github.com/tikv/pd/client.(*tsoClient).Setup in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_client.go:126 +0xc5 goroutine 617 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc001d002d0, 0x9e5) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000112f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001f1e000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 599 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 156 [select]: github.com/pingcap/tidb/util/cpu.(*Observer).Start.func1() /go/src/github.com/pingcap/tidb/util/cpu/cpu.go:76 +0xf5 created by github.com/pingcap/tidb/util/cpu.(*Observer).Start in goroutine 435 /go/src/github.com/pingcap/tidb/util/cpu/cpu.go:69 +0x19b goroutine 573 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7960, {0x657b950, 0xc001216e10}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 648 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc00105a210, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 218 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000e72240) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 488 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 220 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000de0410, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b763f0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 488 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 722 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 706 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 618 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04794b8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000f2dd80?, 0xc001f0a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000f2dd80, {0xc001f0a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000f2dd80, {0xc001f0a000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001e74028, {0xc001f0a000?, 0x0?, 0xce36f2795fe136db?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001f08000, {0xc001f1a040, 0x9, 0x7f8b191ad1d8?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001f08000}, {0xc001f1a040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001f1a040, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc001f08000?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001f1a000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001f1e000, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 599 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 128 [select, 3774 minutes]: github.com/pingcap/tidb/ddl.(*ddl).limitDDLJobs(0xc000b6a3f0) /go/src/github.com/pingcap/tidb/ddl/ddl_worker.go:199 +0x133 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 225 [select]: github.com/tikv/pd/client.(*pdServiceDiscovery).updateServiceModeLoop(0xc00146c3c0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:580 +0x145 created by github.com/tikv/pd/client.(*pdServiceDiscovery).Init in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:514 +0x352 goroutine 639 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04473e0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0010ad400?, 0xc002304000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0010ad400, {0xc002304000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0010ad400, {0xc002304000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e06678, {0xc002304000?, 0xc00125ff80?, 0xc0023e8cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001c4ca20, {0xc00230c040, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001c4ca20}, {0xc00230c040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc00230c040, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001c4ca20?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00230c000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0019aab40, 0xc0000ad840?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 941 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 477 [select]: github.com/tikv/pd/client.(*tsoClient).watchTSDeadline.func1({0x5b7488a, 0x6}, 0xc000a9ade0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:202 +0xbd created by github.com/tikv/pd/client.(*tsoClient).watchTSDeadline in goroutine 223 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:200 +0x17b goroutine 224 [select, 2 minutes]: github.com/tikv/pd/client.(*pdServiceDiscovery).updateMemberLoop(0xc00146c3c0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:548 +0x1d0 created by github.com/tikv/pd/client.(*pdServiceDiscovery).Init in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:513 +0x310 goroutine 454 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6b40, {0x657b950, 0xc001c78e10}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 566 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00124c000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 546 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 456 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6d40, {0x657b950, 0xc001c791d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 457 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6e20, {0x657b950, 0xc001c79450}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 458 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce6e80, {0x657b950, 0xc001c794a0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 472 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001aa6760, {0x657b950, 0xc000c86370}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 190 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000dc0eb0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b8a5b0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 455 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 474 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001aa68a0, {0x657b950, 0xc000c86640}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 462 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 221 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 482 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000d50490, 0x75f) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc001ac1f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001002240) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 475 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 483 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad0479c78, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001d1a680?, 0xc00101a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001d1a680, {0xc00101a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001d1a680, {0xc00101a000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001a94088, {0xc00101a000?, 0x5716260?, 0xc463e45350?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e883c0, {0xc001006040, 0x9, 0x7f8b191aba68?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e883c0}, {0xc001006040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001006040, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e883c0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001006000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001002240, 0xc00145001c?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 475 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 484 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d18870, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001bb110) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 475 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 485 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0007af380, {0x657b950, 0xc000d18960}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 486 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0007af460, {0x657b950, 0xc000d18be0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 487 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0007af4c0, {0x657b950, 0xc000d18c30}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 463 [select, 3774 minutes]: github.com/tikv/pd/client.(*tsoClient).handleDispatcher.func2() /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:381 +0x16f created by github.com/tikv/pd/client.(*tsoClient).handleDispatcher in goroutine 221 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_dispatcher.go:369 +0x1e9 goroutine 489 [select, 2 minutes]: github.com/tikv/pd/client.(*tsoServiceDiscovery).startCheckMemberLoop(0xc00018dba0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_service_discovery.go:254 +0x178 created by github.com/tikv/pd/client.(*tsoServiceDiscovery).Init in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/tso_service_discovery.go:204 +0x38b goroutine 498 [select]: github.com/tikv/pd/client.(*pdServiceDiscovery).memberHealthCheckLoop(0xc00146c3c0) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:603 +0x165 created by github.com/tikv/pd/client.(*pdServiceDiscovery).Init in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/pd_service_discovery.go:515 +0x399 goroutine 499 [select]: github.com/tikv/pd/client.(*client).handleResourceTokenDispatcher(0xc0018924d0, {0x657b950, 0xc000de0730}, 0xc001cd2090) /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/resource_manager_client.go:311 +0x258 created by github.com/tikv/pd/client.(*client).createTokenDispatcher in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/resource_manager_client.go:289 +0x165 goroutine 191 [select]: github.com/iosmanthus/cse-region-client.(*Client).refreshStoresLoop(0xc000e323c0) /go/cache/github.com/iosmanthus/cse-region-client@v0.0.0-20231222080054-384736f92a0a/cse.go:212 +0x85 created by github.com/iosmanthus/cse-region-client.NewClient in goroutine 1 /go/cache/github.com/iosmanthus/cse-region-client@v0.0.0-20231222080054-384736f92a0a/cse.go:116 +0x32b goroutine 203 [select]: github.com/tikv/client-go/v2/internal/client.(*connMonitor).start(0xc000f26900) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:223 +0xbe created by github.com/tikv/client-go/v2/internal/client.(*connMonitor).Start.func1 in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:203 +0x89 goroutine 204 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000c9c20, {0x657b950, 0xc000f30ff0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 205 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000c9d00, {0x657b950, 0xc000f311d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 206 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000c9e20, {0x657b950, 0xc000f31220}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 539 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0c20, {0x657b950, 0xc0010feff0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 208 [select]: go.etcd.io/etcd/client/v3.(*Client).autoSync(0xc0017868c0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:202 +0xb1 created by go.etcd.io/etcd/client/v3.newClient in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:451 +0x110c goroutine 209 [select]: github.com/tikv/client-go/v2/oracle/oracles.(*pdOracle).updateTS(0xc000f2f020, {0x657b8e0, 0x9529b80}, 0x0?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/oracle/oracles/pd.go:249 +0x117 created by github.com/tikv/client-go/v2/oracle/oracles.NewPdOracle in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/oracle/oracles/pd.go:77 +0xe5 goroutine 540 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0d00, {0x657b950, 0xc0010ff270}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 632 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000c875e0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000a62150) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 771 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 530 [select]: github.com/tikv/client-go/v2/internal/locate.(*RegionCache).asyncCheckAndResolveLoop(0xc001935c00, 0x0?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:578 +0x1b2 created by github.com/tikv/client-go/v2/internal/locate.NewRegionCache in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:532 +0x6b6 goroutine 531 [select]: github.com/tikv/client-go/v2/internal/locate.(*RegionCache).asyncUpdateStoreSlowScore(0xc001935c00, 0x0?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:3006 +0xc7 created by github.com/tikv/client-go/v2/internal/locate.NewRegionCache in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:535 +0x77f goroutine 532 [select]: github.com/tikv/client-go/v2/internal/locate.(*RegionCache).cacheGC(0xc001935c00) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:2192 +0x186 created by github.com/tikv/client-go/v2/internal/locate.NewRegionCache in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:540 +0x8a5 goroutine 533 [select, 2 minutes]: github.com/tikv/client-go/v2/internal/locate.(*RegionCache).asyncReportStoreReplicaFlows(0xc001935c00, 0x0?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:3058 +0xc8 created by github.com/tikv/client-go/v2/internal/locate.NewRegionCache in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_cache.go:542 +0x8f4 goroutine 534 [select]: github.com/tikv/client-go/v2/tikv.(*KVStore).runSafePointChecker(0xc00197edc0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikv/kv.go:285 +0x11a created by github.com/tikv/client-go/v2/tikv.NewKVStore in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikv/kv.go:231 +0x545 goroutine 535 [select]: github.com/tikv/client-go/v2/tikv.(*KVStore).safeTSUpdater(0xc00197edc0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikv/kv.go:608 +0x24d created by github.com/tikv/client-go/v2/tikv.NewKVStore in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikv/kv.go:232 +0x585 goroutine 536 [select]: github.com/dgraph-io/ristretto.(*defaultPolicy).processItems(0xc000f27180) /go/cache/github.com/dgraph-io/ristretto@v0.1.1/policy.go:102 +0x85 created by github.com/dgraph-io/ristretto.newDefaultPolicy in goroutine 1 /go/cache/github.com/dgraph-io/ristretto@v0.1.1/policy.go:86 +0x139 goroutine 537 [select]: github.com/dgraph-io/ristretto.(*Cache).processItems(0xc000f2cb00) /go/cache/github.com/dgraph-io/ristretto@v0.1.1/cache.go:452 +0x149 created by github.com/dgraph-io/ristretto.NewCache in goroutine 1 /go/cache/github.com/dgraph-io/ristretto@v0.1.1/cache.go:207 +0x645 goroutine 538 [select]: github.com/pingcap/tidb/store/copr.(*MPPFailedStoreProber).Run.func1() /go/src/github.com/pingcap/tidb/store/copr/mpp_probe.go:203 +0x1dc created by github.com/pingcap/tidb/store/copr.(*MPPFailedStoreProber).Run in goroutine 1 /go/src/github.com/pingcap/tidb/store/copr/mpp_probe.go:196 +0xba goroutine 541 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0d60, {0x657b950, 0xc0010ff2c0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 506 [sync.Cond.Wait, 4 minutes]: sync.runtime_notifyListWait(0xc000ddc790, 0x9be) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc001ccef70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000e72900) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 597 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 543 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0f80, {0x657b950, 0xc0010ff680}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 544 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1060, {0x657b950, 0xc0010ff900}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 545 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e10c0, {0x657b950, 0xc0010ff950}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 567 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04797a0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001284380?, 0xc001224000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001284380, {0xc001224000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001284380, {0xc001224000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e06ae8, {0xc001224000?, 0xc001c4dec0?, 0xc0000b8cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001c4dda0, {0xc001c76ba0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001c4dda0}, {0xc001c76ba0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001c76ba0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001c4dda0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001c76b60) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00124c000, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 546 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 507 [IO wait, 4 minutes]: internal/poll.runtime_pollWait(0x7f8ad0447ba0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001d1a980?, 0xc00136a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001d1a980, {0xc00136a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001d1a980, {0xc00136a000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd2100, {0xc00136a000?, 0x5716260?, 0xc43734bbf0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32720, {0xc000e44580, 0x9, 0x7f8b191ab5b8?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32720}, {0xc000e44580, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000e44580, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e32720?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e44540) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000e72900, 0xc001cd20d8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 597 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 569 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 556 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 547 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e12e0, {0x657b950, 0xc0010ffd10}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 548 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e13c0, {0x657b950, 0xc0010fff90}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 549 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1420, {0x657b950, 0xc001542000}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 867 [chan receive, 3774 minutes]: github.com/pingcap/tidb/domain.(*Domain).StartPlanReplayerHandle.func2() /go/src/github.com/pingcap/tidb/domain/domain.go:2035 +0x153 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 551 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1640, {0x657b950, 0xc0015423c0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 552 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1720, {0x657b950, 0xc001542640}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 553 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1780, {0x657b950, 0xc001542690}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 638 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0019aab40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 941 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 555 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc000f2fbc0, 0xc000edb080) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 556 [select]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc000f2d780, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc000f2d780, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 1 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 570 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0012165f0, {0xc001c7b990, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0012165f0, {0xc001c7b990?, 0xc0016f3b48?, 0xc178aad608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001c7b950, {0xc001c7b990?, 0xc178aad680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc001c7b950}, {0xc001c7b990, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c7cc60, {0xc001c7b990, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc001c7b980, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc001c7cc60, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc001c7b980, {0x7f8ad0162008, 0x9529b80}, 0xc001c7cc60?, {0x0, 0x0}, {0x598ae00, 0xc6a4224840}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00130dad0, {0x598ae00?, 0xc6a4224840}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001c7ca20, 0xc178aadaa8, 0xc178aada98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c7ca20, {0x598ae00?, 0xc6a4224840?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001181170) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001ce7400) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc00197fce0, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 556 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 502 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000e726c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 542 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 503 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0479b80, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000d4d080?, 0xc001330000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000d4d080, {0xc001330000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000d4d080, {0xc001330000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd20d8, {0xc001330000?, 0xc001432c60?, 0xc001a8bcd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32600, {0xc000e444a0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32600}, {0xc000e444a0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000e444a0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc000e32600?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e44460) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000e726c0, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 542 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 504 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000de0c30, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b768c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 542 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 522 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0019fa6c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 523 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04796a8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001c75000?, 0xc0018a6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001c75000, {0xc0018a6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001c75000, {0xc0018a6000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0000ad840, {0xc0018a6000?, 0xc001432780?, 0xc000ce4cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0019c7920, {0xc0019ca3c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0019c7920}, {0xc0019ca3c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0019ca3c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0019c7920?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019ca380) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0019fa6c0, 0x67756265642e231a?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 496 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001002900) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 550 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 497 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04795b0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000e12600?, 0xc001434000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000e12600, {0xc001434000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000e12600, {0xc001434000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001a941b8, {0xc001434000?, 0xc001432300?, 0xc0000b9cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001432000, {0xc0010063c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001432000}, {0xc0010063c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0010063c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001432000?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001006380) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001002900, 0xc001a0b200?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 550 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 578 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001430140, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001bbab0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 550 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 524 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000dc1a40, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b8aaf0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 579 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 556 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 580 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001430460, {0xc00142c940, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001430460, {0xc00142c940?, 0xc001cec228?, 0xc001531608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00142c900, {0xc00142c940?, 0xc001531680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00142c900}, {0xc00142c940, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001001200, {0xc00142c940, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00142c930, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc001001200, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00142c930, {0x7f8ad0162008, 0x9529b80}, 0x175d0000175e?, {0x0, 0x0}, {0x598ae00, 0xc6a42258c0}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce29c0, {0x598ae00?, 0xc6a42258c0}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001000fc0, 0xc001531aa8, 0xc001531a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001000fc0, {0x598ae00?, 0xc6a42258c0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001173730) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc0007afce0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc00197fe40, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 556 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 581 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 556 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 582 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0014308c0, {0xc00142ceb0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0014308c0, {0xc00142ceb0?, 0xc001cec2a0?, 0xc0013f7608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00142ce70, {0xc00142ceb0?, 0xc0013f7680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00142ce70}, {0xc00142ceb0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010017a0, {0xc00142ceb0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00142cea0, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc00142cea0?, 0xc0010017a0, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00142cea0, {0x7f8ad0162008, 0x9529b80}, 0xc0013f7a20?, {0x0, 0x0}, {0x598ae00, 0xc6b0dc3da0}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce2c30, {0x598ae00?, 0xc6b0dc3da0}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001001560, 0xc0013f7aa8, 0xc0013f7a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001001560, {0x598ae00?, 0xc6b0dc3da0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001173860) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc0007afde0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc001556000, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 556 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 583 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 556 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 584 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001430d20, {0xc00142d420, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001430d20, {0xc00142d420?, 0xc001cec318?, 0xc0013f5608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00142d3e0, {0xc00142d420?, 0xc0013f5680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00142d3e0}, {0xc00142d420, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001001d40, {0xc00142d420, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00142d410, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc001001d40, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00142d410, {0x7f8ad0162008, 0x9529b80}, 0xc0013f5a20?, {0x0, 0x0}, {0x598ae00, 0xc70ef10780}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce2dd0, {0x598ae00?, 0xc70ef10780}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001001b00, 0xc0013f5aa8, 0xc0013f5a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001001b00, {0x598ae00?, 0xc70ef10780?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001173a10) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc0007aff00) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc00197fb80, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 556 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 631 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04477c0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0022dee80?, 0xc001bcc000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0022dee80, {0xc001bcc000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0022dee80, {0xc001bcc000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc00198f988, {0xc001bcc000?, 0xc00244dc80?, 0xc00253dcd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001accfc0, {0xc0019ca580, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001accfc0}, {0xc0019ca580, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0019ca580, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001accfc0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019ca540) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001bac480, 0xc0019f001c?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 748 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 633 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*JobManager).jobLoop(0xc001f2e000) /go/src/github.com/pingcap/tidb/ttl/ttlworker/job_manager.go:172 +0x554 github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x2d40b8c?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 882 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 577 [select]: go.etcd.io/etcd/client/v3.(*Client).autoSync(0xc0013a3340) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:202 +0xb1 created by go.etcd.io/etcd/client/v3.newClient in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:451 +0x110c goroutine 148779937 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0b8e15760, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 634 [chan receive, 3774 minutes]: github.com/pingcap/tidb/ttl/client.(*etcdClient).WatchCommand.func1() /go/src/github.com/pingcap/tidb/ttl/client/command.go:273 +0x147 created by github.com/pingcap/tidb/ttl/client.(*etcdClient).WatchCommand in goroutine 633 /go/src/github.com/pingcap/tidb/ttl/client/command.go:265 +0xa5 goroutine 636 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc001bee160, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 866 [select]: github.com/pingcap/tidb/domain.(*Domain).StartPlanReplayerHandle.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:2013 +0x15d github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 635 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc001bee0b0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 129 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc00193ab00, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 594 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7e00, {0x657b950, 0xc001217860}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 595 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7ea0, {0x657b950, 0xc001217a40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 596 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001ce7f00, {0x657b950, 0xc001217a90}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 661 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001e83c70, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b68000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 775 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 620 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc001e72640}, 0x0?, 0xc001e7e4e0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 600 [select]: go.etcd.io/etcd/client/v3.(*Client).autoSync(0xc0013a3a40) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:202 +0xb1 created by go.etcd.io/etcd/client/v3.newClient in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:451 +0x110c goroutine 593 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04790d8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001ba6a00?, 0xc001f84000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001ba6a00, {0xc001f84000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001ba6a00, {0xc001f84000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001a942a8, {0xc001f84000?, 0x0?, 0xc001440800?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001433140, {0xc0010064a0, 0x9, 0x7f8b191abf18?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001433140}, {0xc0010064a0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0010064a0, 0x9, 0xc5bad93848?}, {0x65496a0?, 0xc001433140?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001006460) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001003200, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 576 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 642 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001431220, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000c600e0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 576 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 508 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000de0ff0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b76a10) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 597 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 644 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001003440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 574 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 645 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04793c0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000d4d600?, 0xc001f9c000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000d4d600, {0xc001f9c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000d4d600, {0xc001f9c000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001a942b8, {0xc001f9c000?, 0x0?, 0xc03aa07b34?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001433260, {0xc001006580, 0x9, 0x7f8b191acd28?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001433260}, {0xc001006580, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001006580, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc001433260?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001006540) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001003440, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 574 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 646 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001431360, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000c60150) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 574 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 510 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000ddc8d0, 0x1fd3) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc001ccf770?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000e72d80) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 575 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 511 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad0447aa8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001d1aa00?, 0xc002010000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001d1aa00, {0xc002010000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001d1aa00, {0xc002010000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd2118, {0xc002010000?, 0x18?, 0xc6f3584f30?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32840, {0xc000e44660, 0x9, 0x7f8b191ab108?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32840}, {0xc000e44660, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000e44660, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e32840?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e44620) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000e72d80, 0xc001cd20d8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 575 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 512 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000de1220, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b76af0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 575 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 602 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc0016a56d0, 0x9da) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc001ac3f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00124cb40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 598 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 603 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad04792c8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000c1a480?, 0xc001e88000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000c1a480, {0xc001e88000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000c1a480, {0xc001e88000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e07100, {0xc001e88000?, 0x5716260?, 0xc5793dab10?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001c02360, {0xc001c76d60, 0x9, 0x7f8b191aba68?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001c02360}, {0xc001c76d60, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001c76d60, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc001c02360?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001c76d20) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00124cb40, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 598 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 604 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001e827d0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b6a540) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 598 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 619 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001e72230, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b72310) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 599 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 622 [select]: go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc000dc8960) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:550 +0x75 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:302 +0x96 goroutine 623 [chan receive, 176 minutes]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 605 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 621 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 606 [select]: go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc000dc8960, {0x6598810, 0xc0016df0b0}) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:590 +0x1cf created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 621 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:500 +0x2bb goroutine 624 [select]: github.com/pingcap/tidb/domain.(*Domain).serverIDKeeper(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:2784 +0x26a created by github.com/pingcap/tidb/domain.(*Domain).Init in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:1166 +0x8d0 goroutine 607 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc001e82c80}, 0x6598810?, 0xc001223bc0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 608 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 625 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc001e73270}, 0xc000fe9e50?, 0xc001e7e9c0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 674 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 609 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc001e839a0}, 0xc00147e1c0?, 0xc001eb6480) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 690 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 691 [select, 3774 minutes]: context.(*cancelCtx).propagateCancel.func2() /usr/local/go/src/context/context.go:505 +0x98 created by context.(*cancelCtx).propagateCancel in goroutine 1 /usr/local/go/src/context/context.go:504 +0x395 goroutine 692 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc0016eca90) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:545 +0x33b created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:292 +0x2ff goroutine 647 [select, 411 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001e83b30, {0xc001ea1cf0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001e83b30, {0xc001ea1cf0?, 0xc0016f3e78?, 0xc001d079f0?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001ea1cb0, {0xc001ea1cf0?, 0xc001d07a68?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc001ea1cb0}, {0xc001ea1cf0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001eb0ea0, {0xc001ea1cf0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc001ea1ce0, 0x7fffffff) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc001465c48?, 0xc001eb0ea0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc001ea1ce0, {0x7f8ad0162008, 0x9529b80}, 0xc001c4dda0?, {0x0, 0x0}, {0x59fbf00, 0xc0b9b4a0e0}, 0x0?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0016ecc30, {0x59fbf00?, 0xc0b9b4a0e0}, 0xc001d07de0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d07de0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001eb0c60, 0xc001d07e90, 0xc001d07e80) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001eb0c60, {0x59fbf00?, 0xc0b9b4a0e0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0016df890) /go/cache/go.etcd.io/etcd/api/v3@v3.5.10/etcdserverpb/rpc.pb.go:6714 +0x46 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0016eca90, {0x65988c0, 0xc0016df890}) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:766 +0x5c created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:921 +0x598 goroutine 694 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 692 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 651 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc0023b6000}, 0x0?, 0xc00142f8c0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 675 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc002346370}, 0x0?, 0xc001e7eea0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 676 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 513 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 706 [select]: github.com/pingcap/tidb/ddl.(*ddl).startDispatchLoop(0xc000b6a3f0) /go/src/github.com/pingcap/tidb/ddl/job_table.go:283 +0x6af github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 707 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0011f26e0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 652 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 653 [select, 411 minutes]: github.com/pingcap/tidb/owner.(*ownerManager).watchOwner(0xc000dc8d20, {0x657b950?, 0xc002346be0?}, 0xc0023a3a10, {0xc0ab97f830, 0x23}) /go/src/github.com/pingcap/tidb/owner/manager.go:436 +0x2e5 github.com/pingcap/tidb/owner.(*ownerManager).campaignLoop(0xc000dc8d20, 0xc0023a3a10) /go/src/github.com/pingcap/tidb/owner/manager.go:296 +0x839 created by github.com/pingcap/tidb/owner.(*ownerManager).CampaignOwner in goroutine 1 /go/src/github.com/pingcap/tidb/owner/manager.go:189 +0x2b9 goroutine 654 [select]: github.com/pingcap/tidb/ddl.(*ddl).PollTiFlashRoutine(0xc000b6a3f0) /go/src/github.com/pingcap/tidb/ddl/ddl_tiflash_api.go:649 +0x28d github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 655 [select]: github.com/pingcap/tidb/domain.(*Domain).loadSchemaInLoop(0xc001736000, {0x657b950, 0xc001e823c0}, 0xc001322580?) /go/src/github.com/pingcap/tidb/domain/domain.go:873 +0x1f4 github.com/pingcap/tidb/domain.(*Domain).Init.func3() /go/src/github.com/pingcap/tidb/domain/domain.go:1216 +0x25 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 656 [select]: github.com/pingcap/tidb/domain.(*Domain).mdlCheckLoop(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:788 +0x10a github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 657 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).topNSlowQueryLoop(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:650 +0x169 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 708 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).infoSyncerKeeper(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:683 +0x151 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 709 [select, 3774 minutes]: github.com/pingcap/tidb/domain.(*Domain).globalConfigSyncerKeeper(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:707 +0xe8 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 710 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).topologySyncerKeeper(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:729 +0x134 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 711 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).closestReplicaReadCheckLoop(0x0?, {0x657b950?, 0xc001e823c0?}, {0x65fd1b0, 0xc0000c9940}) /go/src/github.com/pingcap/tidb/domain/domain.go:1283 +0x2ba github.com/pingcap/tidb/domain.(*Domain).Init.func4() /go/src/github.com/pingcap/tidb/domain/domain.go:1228 +0x27 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 723 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc000b3fea0}, 0x14?, 0xc001303b60) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 724 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 725 [select, 411 minutes]: github.com/pingcap/tidb/owner.(*ownerManager).watchOwner(0xc000b1ae60, {0x657b950?, 0xc001543180?}, 0xc0020fe390, {0xc0b9992450, 0x26}) /go/src/github.com/pingcap/tidb/owner/manager.go:436 +0x2e5 github.com/pingcap/tidb/owner.(*ownerManager).campaignLoop(0xc000b1ae60, 0xc0020fe390) /go/src/github.com/pingcap/tidb/owner/manager.go:296 +0x839 created by github.com/pingcap/tidb/owner.(*ownerManager).CampaignOwner in goroutine 1 /go/src/github.com/pingcap/tidb/owner/manager.go:189 +0x2b9 goroutine 148778747 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0af899ad0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 148778825 [chan receive, 411 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 653 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 738 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc00105b6b0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 739 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 740 [select, 3774 minutes]: github.com/pingcap/tidb/br/pkg/streamhelper.AdvancerExt.startListen.func3() /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer_cliext.go:151 +0x157 created by github.com/pingcap/tidb/br/pkg/streamhelper.AdvancerExt.startListen in goroutine 1 /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer_cliext.go:148 +0x2e6 goroutine 741 [select, 3774 minutes]: github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).StartTaskListener.func1() /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer.go:265 +0x107 created by github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).StartTaskListener in goroutine 1 /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer.go:262 +0x245 goroutine 742 [select]: github.com/pingcap/tidb/br/pkg/streamhelper/daemon.(*OwnerDaemon).Begin.func1() /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/daemon/owner_daemon.go:85 +0x23d github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 5322551 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc05cfc2480) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322541 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 794 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 781 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 666 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 781 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 667 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000b16550, {0xc00259dba0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b16550, {0xc00259dba0?, 0xc000eaab70?, 0xc0019cf608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00259db60, {0xc00259dba0?, 0xc0019cf680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00259db60}, {0xc00259dba0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000f68fc0, {0xc00259dba0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00259db90, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc00259db90?, 0xc000f68fc0, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00259db90, {0x7f8ad0162008, 0x9529b80}, 0xc0019cfa20?, {0x0, 0x0}, {0x598ae00, 0xc663756120}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00252fe10, {0x598ae00?, 0xc663756120}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc000f68d80, 0xc0019cfaa8, 0xc0019cfa98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc000f68d80, {0x598ae00?, 0xc663756120?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc002452b20) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001ba4720) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc00259a000, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 781 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 669 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000b16910, {0xc00259ded0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b16910, {0xc00259ded0?, 0xc000eaabd0?, 0xc001535608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00259de90, {0xc00259ded0?, 0xc001535680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00259de90}, {0xc00259ded0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000f69560, {0xc00259ded0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00259dec0, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc000f69560, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00259dec0, {0x7f8ad0162008, 0x9529b80}, 0xc001535a20?, {0x0, 0x0}, {0x598ae00, 0xc7209f6060}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007fcf70, {0x598ae00?, 0xc7209f6060}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc000f69320, 0xc001535aa8, 0xc001535a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc000f69320, {0x598ae00?, 0xc7209f6060?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc002452c20) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001ba47c0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc0022d5b80, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 781 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 810 [select]: github.com/pingcap/tidb/domain.(*Domain).updateStatsWorker(0xc001736000, {0x65fde38?, 0xc000a6af00?}, {0x659e440?, 0xc0019dee60?}) /go/src/github.com/pingcap/tidb/domain/domain.go:2386 +0x3b4 github.com/pingcap/tidb/domain.(*Domain).UpdateTableStatsLoop.func2() /go/src/github.com/pingcap/tidb/domain/domain.go:2218 +0x27 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 761 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 633 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 745 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023adda0, {0x657b950, 0xc0023b66e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 746 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023ade80, {0x657b950, 0xc0023b6960}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 747 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023adee0, {0x657b950, 0xc0023b69b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 762 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlScanWorker).loop(0xc001fc4bd0) /go/src/github.com/pingcap/tidb/ttl/ttlworker/scan.go:315 +0x165 github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0xc000c2b810?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 749 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa120, {0x657b950, 0xc0023b6d70}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 750 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa200, {0x657b950, 0xc0023b6ff0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 751 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa260, {0x657b950, 0xc0023b7040}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 737 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 634 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 660 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000c80900) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 771 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 630 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001bac480) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 748 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 783 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 781 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 772 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa520, {0x657b950, 0xc0023b7680}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 773 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa600, {0x657b950, 0xc0023b7900}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 774 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa660, {0x657b950, 0xc0023b7950}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 670 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 776 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023fa960, {0x657b950, 0xc0023b7f40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 777 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023faa40, {0x657b950, 0xc0025941e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 778 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0023faaa0, {0x657b950, 0xc002594230}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 743 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 736 [select, 2 minutes]: github.com/pingcap/tidb/store/gcworker.(*GCWorker).start(0xc002124000, {0x657b918, 0xc001baa0f0}, 0x223a224f227b3a22?) /go/src/github.com/pingcap/tidb/store/gcworker/gc_worker.go:255 +0x2f0 created by github.com/pingcap/tidb/store/gcworker.(*GCWorker).Start in goroutine 1 /go/src/github.com/pingcap/tidb/store/gcworker/gc_worker.go:144 +0x112 goroutine 780 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc0023b4d80, 0xc00142fda0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 743 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 781 [select]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc0022de880, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc0022de880, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 743 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 714 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015434a0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000dfe9a0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 779 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 811 [select]: github.com/pingcap/tidb/domain.(*Domain).autoAnalyzeWorker(0xc001736000, {0x659e440, 0xc0019dee60}) /go/src/github.com/pingcap/tidb/domain/domain.go:2450 +0xe5 github.com/pingcap/tidb/domain.(*Domain).UpdateTableStatsLoop.func3() /go/src/github.com/pingcap/tidb/domain/domain.go:2219 +0x1f github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 784 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000c861e0, {0xc00245a310, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c861e0, {0xc00245a310?, 0xc00230e000?, 0xc001d09608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00245a2d0, {0xc00245a310?, 0xc001d09680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00245a2d0}, {0xc00245a310, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0013b47e0, {0xc00245a310, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00245a300, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc001d09838?, 0xc0013b47e0, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00245a300, {0x7f8ad0162008, 0x9529b80}, 0xc001d09a20?, {0x0, 0x0}, {0x598ae00, 0xc786064720}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001f200d0, {0x598ae00?, 0xc786064720}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc0013b4120, 0xc001d09aa8, 0xc001d09a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc0013b4120, {0x598ae00?, 0xc786064720?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc0023e5a50) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001aa6080) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc0022d5ce0, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 781 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 671 [select]: github.com/pingcap/tidb/domain.(*Domain).WatchTiFlashComputeNodeChange.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:1745 +0x156 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 763 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlScanWorker).loop(0xc001fc4c60) /go/src/github.com/pingcap/tidb/ttl/ttlworker/scan.go:315 +0x165 github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0xc0023be7d0?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 757 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04791d0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000e13680?, 0xc002486000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000e13680, {0xc002486000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000e13680, {0xc002486000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0010c25d0, {0xc002486000?, 0xc00244d9e0?, 0xc002541cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc002482360, {0xc000f349e0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc002482360}, {0xc000f349e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000f349e0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc002482360?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f349a0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f65b00, 0xc0020fe390?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 779 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 735 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04479b0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc002445980?, 0xc002608000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc002445980, {0xc002608000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc002445980, {0xc002608000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001450260, {0xc002608000?, 0xc001acc060?, 0xc000ce3cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000a9b980, {0xc000aec200, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000a9b980}, {0xc000aec200, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc000aec200, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc000a9b980?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000aec1c0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000c80900, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 771 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 697 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00124dd40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 775 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 698 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04478b8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001eacd80?, 0xc001ed0000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001eacd80, {0xc001ed0000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001eacd80, {0xc001ed0000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e07260, {0xc001ed0000?, 0xc001acd980?, 0xc000cf2cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001c03860, {0xc001c77000, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001c03860}, {0xc001c77000, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001c77000, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001c03860?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001c76fc0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00124dd40, 0xc000a80001?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 775 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 758 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001725e00, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000b72000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 748 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 785 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc000c86280}, 0xc0022de8a8?, 0xc000cea2a0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 786 [chan receive, 2 minutes]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 787 [select, 411 minutes]: github.com/pingcap/tidb/owner.(*ownerManager).watchOwner(0xc001e7c1e0, {0x657b950?, 0xc002454140?}, 0xc00245a690, {0xc0b5ada840, 0x25}) /go/src/github.com/pingcap/tidb/owner/manager.go:436 +0x2e5 github.com/pingcap/tidb/owner.(*ownerManager).campaignLoop(0xc001e7c1e0, 0xc00245a690) /go/src/github.com/pingcap/tidb/owner/manager.go:296 +0x839 created by github.com/pingcap/tidb/owner.(*ownerManager).CampaignOwner in goroutine 1 /go/src/github.com/pingcap/tidb/owner/manager.go:189 +0x2b9 goroutine 788 [select]: github.com/pingcap/tidb/domain.(*Domain).LoadBindInfoLoop.(*Domain).globalBindHandleWorkerLoop.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:1824 +0x170 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 789 [select]: github.com/pingcap/tidb/domain.(*Domain).LoadBindInfoLoop.(*Domain).handleEvolvePlanTasksLoop.func2() /go/src/github.com/pingcap/tidb/domain/domain.go:1861 +0xea github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 809 [select, 411 minutes]: github.com/pingcap/tidb/owner.(*ownerManager).watchOwner(0xc0019dee60, {0x657b950?, 0xc001c783c0?}, 0xc0012e5140, {0xc0b136aa20, 0x22}) /go/src/github.com/pingcap/tidb/owner/manager.go:436 +0x2e5 github.com/pingcap/tidb/owner.(*ownerManager).campaignLoop(0xc0019dee60, 0xc0012e5140) /go/src/github.com/pingcap/tidb/owner/manager.go:296 +0x839 created by github.com/pingcap/tidb/owner.(*ownerManager).CampaignOwner in goroutine 1 /go/src/github.com/pingcap/tidb/owner/manager.go:189 +0x2b9 goroutine 808 [chan receive, 2 minutes]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 668 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 781 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 807 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc000dc8960, {0x657b950, 0xc001cd0870}, 0xc001725ef0?, 0xc002426ea0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 795 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002505180, {0xc000eecc40, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002505180, {0xc000eecc40?, 0xc00230e4b0?, 0xc001537608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eecc00, {0xc000eecc40?, 0xc001537680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc000eecc00}, {0xc000eecc40, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0011c38c0, {0xc000eecc40, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc000eecc30, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc0011c38c0, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc000eecc30, {0x7f8ad0162008, 0x9529b80}, 0x175d0000175e?, {0x0, 0x0}, {0x598ae00, 0xc786064ba0}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001f21520, {0x598ae00?, 0xc786064ba0}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc0011c3680, 0xc001537aa8, 0xc001537a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc0011c3680, {0x598ae00?, 0xc786064ba0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc002532770) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001aa6a40) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc0022d5e40, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 781 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 164440302 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed79440, {0x657b950, 0xc6a3c007d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 1084 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad0446f08, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0022cc780?, 0xc00280a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0022cc780, {0xc00280a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0022cc780, {0xc00280a000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e06968, {0xc00280a000?, 0x5716260?, 0xc583a01bc0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc002288420, {0xc00230c2e0, 0x9, 0x7f8b191aba68?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc002288420}, {0xc00230c2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc00230c2e0, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc002288420?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00230c2a0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0019abb00, 0xc0021a2750?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1111 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 1117 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000ed9440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1113 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 5322059 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc05c8143c0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc058dcc5b0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322012 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 1154 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad0446d18, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000b67c80?, 0xc0026b6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000b67c80, {0xc0026b6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000b67c80, {0xc0026b6000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001e74bd0, {0xc0026b6000?, 0x5716260?, 0xc78fe76150?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc00217ec00, {0xc0026aa2e0, 0x9, 0x7f8b191abf18?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc00217ec00}, {0xc0026aa2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0026aa2e0, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc00217ec00?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0026aa2a0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001badb00, 0xc000b80001?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1112 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 1155 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00269e9b0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000ff6770) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1112 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 1085 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00235af00, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001190230) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1111 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164440288 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc41d4fc370, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0df42e1c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440296 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 688 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc001a98580, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 689 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 834 [select, 4 minutes]: github.com/pingcap/tidb/domain.(*Domain).LoadPrivilegeLoop.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:1635 +0x16f github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 108039 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc005d48a80, 0xc0058eaae0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 107997 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 148782025 [chan receive, 411 minutes]: github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).OnBecomeOwner.func1() /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer_daemon.go:39 +0x2c created by github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).OnBecomeOwner in goroutine 742 /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer_daemon.go:38 +0xb1 goroutine 806 [chan receive, 3774 minutes]: github.com/pingcap/tidb/domain.(*planReplayerTaskDumpWorker).run(0xc001455680) /go/src/github.com/pingcap/tidb/domain/plan_replayer.go:382 +0xcb created by github.com/pingcap/tidb/domain.(*Domain).StartPlanReplayerHandle.func2 in goroutine 867 /go/src/github.com/pingcap/tidb/domain/domain.go:2033 +0xe5 goroutine 805 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0018c1600, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 832 [chan receive, 3774 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 833 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).LoadSysVarCacheLoop.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:1685 +0x16f github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 850 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc001a568f0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 868 [select, 4 minutes]: github.com/pingcap/tidb/session.bootstrapSessionImpl.(*Domain).DumpFileGcCheckerLoop.func4() /go/src/github.com/pingcap/tidb/domain/domain.go:2067 +0x10f github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 869 [select, 2 minutes]: github.com/pingcap/tidb/domain.(*Domain).StartHistoricalStatsWorker.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:2098 +0x134 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 870 [select]: github.com/pingcap/tidb/domain.(*Domain).loadStatsWorker(0xc001736000) /go/src/github.com/pingcap/tidb/domain/domain.go:2287 +0x166 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 812 [select, 56 minutes]: github.com/pingcap/tidb/domain.(*Domain).gcAnalyzeHistory(0xc001736000, {0x659e440, 0xc0019dee60}) /go/src/github.com/pingcap/tidb/domain/domain.go:2471 +0xf9 github.com/pingcap/tidb/domain.(*Domain).UpdateTableStatsLoop.func4() /go/src/github.com/pingcap/tidb/domain/domain.go:2220 +0x1f github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 813 [select, 3774 minutes]: github.com/pingcap/tidb/statistics/handle.(*Handle).drainColTask(0xc001f30f20, 0xc0012c80c0) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:410 +0xff github.com/pingcap/tidb/statistics/handle.(*Handle).HandleOneTask(0x0?, 0x0?, 0x0?, {0x7f8ad030b238, 0xc000a6b400}, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:224 +0xb6 github.com/pingcap/tidb/statistics/handle.(*Handle).SubLoadWorker(0xc001f30f20, {0x65fde38?, 0xc000a6b400}, 0x0?, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:200 +0xec created by github.com/pingcap/tidb/domain.(*Domain).StartLoadStatsSubWorkers in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:2230 +0x45 goroutine 814 [select, 3774 minutes]: github.com/pingcap/tidb/statistics/handle.(*Handle).drainColTask(0xc001f30f20, 0xc0012c80c0) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:410 +0xff github.com/pingcap/tidb/statistics/handle.(*Handle).HandleOneTask(0x0?, 0x0?, 0x0?, {0x7f8ad030b238, 0xc000af0500}, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:224 +0xb6 github.com/pingcap/tidb/statistics/handle.(*Handle).SubLoadWorker(0xc001f30f20, {0x65fde38?, 0xc000af0500}, 0x0?, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:200 +0xec created by github.com/pingcap/tidb/domain.(*Domain).StartLoadStatsSubWorkers in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:2230 +0x45 goroutine 815 [select, 3774 minutes]: github.com/pingcap/tidb/statistics/handle.(*Handle).drainColTask(0xc001f30f20, 0xc0012c80c0) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:410 +0xff github.com/pingcap/tidb/statistics/handle.(*Handle).HandleOneTask(0x0?, 0x0?, 0x0?, {0x7f8ad030b238, 0xc000af0780}, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:224 +0xb6 github.com/pingcap/tidb/statistics/handle.(*Handle).SubLoadWorker(0xc001f30f20, {0x65fde38?, 0xc000af0780}, 0x0?, 0xc002095600?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:200 +0xec created by github.com/pingcap/tidb/domain.(*Domain).StartLoadStatsSubWorkers in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:2230 +0x45 goroutine 816 [select, 3774 minutes]: github.com/pingcap/tidb/statistics/handle.(*Handle).drainColTask(0xc001f30f20, 0xc0012c80c0) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:410 +0xff github.com/pingcap/tidb/statistics/handle.(*Handle).HandleOneTask(0x0?, 0x0?, 0x0?, {0x7f8ad030b238, 0xc000af0a00}, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:224 +0xb6 github.com/pingcap/tidb/statistics/handle.(*Handle).SubLoadWorker(0xc001f30f20, {0x65fde38?, 0xc000af0a00}, 0x1e65687?, 0xc001eb001c?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:200 +0xec created by github.com/pingcap/tidb/domain.(*Domain).StartLoadStatsSubWorkers in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:2230 +0x45 goroutine 817 [select, 3774 minutes]: github.com/pingcap/tidb/statistics/handle.(*Handle).drainColTask(0xc001f30f20, 0xc0012c80c0) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:410 +0xff github.com/pingcap/tidb/statistics/handle.(*Handle).HandleOneTask(0x0?, 0x0?, 0x0?, {0x7f8ad030b238, 0xc000af0c80}, 0x0?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:224 +0xb6 github.com/pingcap/tidb/statistics/handle.(*Handle).SubLoadWorker(0xc001f30f20, {0x65fde38?, 0xc000af0c80}, 0xc000dc8d20?, 0xc0023a3a10?) /go/src/github.com/pingcap/tidb/statistics/handle/handle_hist.go:200 +0xec created by github.com/pingcap/tidb/domain.(*Domain).StartLoadStatsSubWorkers in goroutine 1 /go/src/github.com/pingcap/tidb/domain/domain.go:2230 +0x45 goroutine 882 [chan receive, 3774 minutes]: github.com/pingcap/tidb/domain.(*Domain).StartTTLJobManager.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:2830 +0xa7 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 883 [select, 4 minutes]: github.com/pingcap/tidb/domain.(*Domain).LoadSigningCertLoop.func1() /go/src/github.com/pingcap/tidb/domain/domain.go:2570 +0xa6 github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 764 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlScanWorker).loop(0xc001fc4cf0) /go/src/github.com/pingcap/tidb/ttl/ttlworker/scan.go:315 +0x165 github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x6574616c6c6f4322?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 765 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlScanWorker).loop(0xc001fc4d80) /go/src/github.com/pingcap/tidb/ttl/ttlworker/scan.go:315 +0x165 github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x735f727078655f64?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 766 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlDeleteWorker).loop(0xc002333b00) /go/src/github.com/pingcap/tidb/ttl/ttlworker/del.go:281 +0x29a github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x3a2277656976222c?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 767 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlDeleteWorker).loop(0xc002333b80) /go/src/github.com/pingcap/tidb/ttl/ttlworker/del.go:281 +0x29a github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x3a226574616c6c6f?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 768 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlDeleteWorker).loop(0xc002333c00) /go/src/github.com/pingcap/tidb/ttl/ttlworker/del.go:281 +0x29a github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x5f6e696769726f22?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 769 [select]: github.com/pingcap/tidb/ttl/ttlworker.(*ttlDeleteWorker).loop(0xc002333c80) /go/src/github.com/pingcap/tidb/ttl/ttlworker/del.go:281 +0x29a github.com/pingcap/tidb/ttl/ttlworker.(*baseWorker).loop(0x5f544c5541464544?) /go/src/github.com/pingcap/tidb/ttl/ttlworker/worker.go:134 +0x66 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 633 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 4284 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc02f3b8490, 0x22c7) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc02ba71f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc02e395440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4310 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 1110 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001a7a5c0, {0x657b950, 0xc00252bae0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 959 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 1011 [select, 3774 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024f9560) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:327 +0x7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:342 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0024f69c0, {0x5989ac0?, 0xc0024d9d10}, 0xc000b02750?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1070 +0xba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc0024f9320, 0xc0024b8e80, 0xc0024b8e70) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:776 +0x13a google.golang.org/grpc.(*clientStream).RecvMsg(0xc0024f9320, {0x5989ac0?, 0xc0024d9d10?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/meta_storagepb.(*metaStorageWatchClient).Recv(0xc0007a9da0) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/meta_storagepb/meta_storagepb.pb.go:988 +0x46 github.com/tikv/pd/client.(*client).Watch.func1() /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/meta_storage_client.go:209 +0x9d created by github.com/tikv/pd/client.(*client).Watch in goroutine 983 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/meta_storage_client.go:204 +0x2e9 goroutine 1010 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 983 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 961 [IO wait, 3774 minutes]: internal/poll.runtime_pollWait(0x7f8ad04471f0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00013bd80?, 0x1?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00013bd80) /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac net.(*netFD).accept(0xc00013bd80) /usr/local/go/src/net/fd_unix.go:172 +0x29 net.(*UnixListener).accept(0xc0024e5500?) /usr/local/go/src/net/unixsock_posix.go:172 +0x16 net.(*UnixListener).Accept(0xc001e7aea0) /usr/local/go/src/net/unixsock.go:260 +0x30 github.com/pingcap/tidb/server.(*Server).startNetworkListener(0xc000a66c80, {0x6566150, 0xc001e7aea0}, 0x1, 0x5?) /go/src/github.com/pingcap/tidb/server/server.go:460 +0x8a created by github.com/pingcap/tidb/server.(*Server).Run in goroutine 1 /go/src/github.com/pingcap/tidb/server/server.go:446 +0x1ca goroutine 960 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04474d8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00013bd00?, 0xc001440800?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00013bd00) /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac net.(*netFD).accept(0xc00013bd00) /usr/local/go/src/net/fd_unix.go:172 +0x29 net.(*TCPListener).accept(0xc001ba5880) /usr/local/go/src/net/tcpsock_posix.go:152 +0x1e net.(*TCPListener).Accept(0xc001ba5880) /usr/local/go/src/net/tcpsock.go:315 +0x30 github.com/pingcap/tidb/server.(*Server).startNetworkListener(0xc000a66c80, {0x6566120, 0xc001ba5880}, 0x0, 0x5?) /go/src/github.com/pingcap/tidb/server/server.go:460 +0x8a created by github.com/pingcap/tidb/server.(*Server).Run in goroutine 1 /go/src/github.com/pingcap/tidb/server/server.go:445 +0x13b goroutine 959 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad04470f8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00013bf00?, 0x4?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00013bf00) /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac net.(*netFD).accept(0xc00013bf00) /usr/local/go/src/net/fd_unix.go:172 +0x29 net.(*TCPListener).accept(0xc001ba58e0) /usr/local/go/src/net/tcpsock_posix.go:152 +0x1e net.(*TCPListener).Accept(0xc001ba58e0) /usr/local/go/src/net/tcpsock.go:315 +0x30 github.com/soheilhy/cmux.(*cMux).Serve(0xc002662280) /go/cache/github.com/soheilhy/cmux@v0.1.5/cmux.go:170 +0x97 github.com/pingcap/tidb/server.(*Server).startStatusServerAndRPCServer(0xc000a66c80, 0xc000daf680) /go/src/github.com/pingcap/tidb/server/http_status.go:529 +0xe1e github.com/pingcap/tidb/server.(*Server).startHTTPServer(0xc000a66c80) /go/src/github.com/pingcap/tidb/server/http_status.go:471 +0x1b3c created by github.com/pingcap/tidb/server.(*Server).startStatusHTTP in goroutine 1 /go/src/github.com/pingcap/tidb/server/http_status.go:66 +0x4f goroutine 957 [select]: github.com/pingcap/tidb/util/topsql/reporter.(*SingleTargetDataSink).run(0xc000e04730) /go/src/github.com/pingcap/tidb/util/topsql/reporter/single_target.go:116 +0x11e github.com/pingcap/tidb/util/topsql/reporter.(*SingleTargetDataSink).recoverRun(0xc000e04730) /go/src/github.com/pingcap/tidb/util/topsql/reporter/single_target.go:97 +0x51 created by github.com/pingcap/tidb/util/topsql/reporter.(*SingleTargetDataSink).Start in goroutine 1 /go/src/github.com/pingcap/tidb/util/topsql/reporter/single_target.go:82 +0x2a5 goroutine 956 [select, 2 minutes]: github.com/pingcap/tidb/util/topsql/reporter.(*RemoteTopSQLReporter).reportWorker(0xc000f0af30) /go/src/github.com/pingcap/tidb/util/topsql/reporter/reporter.go:262 +0xef created by github.com/pingcap/tidb/util/topsql/reporter.(*RemoteTopSQLReporter).Start in goroutine 1 /go/src/github.com/pingcap/tidb/util/topsql/reporter/reporter.go:112 +0xa5 goroutine 12509694 [select, 1213 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0ea7a5810, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0e9219dc0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509848 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 955 [select, 2 minutes]: github.com/pingcap/tidb/util/topsql/reporter.(*RemoteTopSQLReporter).collectWorker(0xc000f0af30) /go/src/github.com/pingcap/tidb/util/topsql/reporter/reporter.go:176 +0x185 created by github.com/pingcap/tidb/util/topsql/reporter.(*RemoteTopSQLReporter).Start in goroutine 1 /go/src/github.com/pingcap/tidb/util/topsql/reporter/reporter.go:111 +0x65 goroutine 954 [select]: github.com/pingcap/tidb/util/topsql/collector.(*SQLCPUCollector).collectSQLCPULoop(0xc001040ec0) /go/src/github.com/pingcap/tidb/util/topsql/collector/cpu.go:120 +0x1a7 created by github.com/pingcap/tidb/util/topsql/collector.(*SQLCPUCollector).Start in goroutine 1 /go/src/github.com/pingcap/tidb/util/topsql/collector/cpu.go:82 +0xb8 goroutine 958 [select]: github.com/pingcap/tidb/util/topsql/stmtstats.(*aggregator).run(0x94f6d00) /go/src/github.com/pingcap/tidb/util/topsql/stmtstats/aggregator.go:67 +0xe5 created by github.com/pingcap/tidb/util/topsql/stmtstats.(*aggregator).start in goroutine 1 /go/src/github.com/pingcap/tidb/util/topsql/stmtstats/aggregator.go:56 +0xcf goroutine 1109 [select]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001a7a560, {0x657b950, 0xc00252ba90}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 959 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 1128 [chan receive]: go.etcd.io/etcd/client/v3/concurrency.NewSession.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:66 +0x74 created by go.etcd.io/etcd/client/v3/concurrency.NewSession in goroutine 959 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/concurrency/session.go:64 +0x267 goroutine 953 [chan receive, 3774 minutes]: github.com/pingcap/tidb/util/signal.SetupSignalHandler.func2() /go/src/github.com/pingcap/tidb/util/signal/signal_posix.go:53 +0x37 created by github.com/pingcap/tidb/util/signal.SetupSignalHandler in goroutine 1 /go/src/github.com/pingcap/tidb/util/signal/signal_posix.go:52 +0x186 goroutine 164440286 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0deac8900) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440296 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 1083 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000d510d0, 0x9e4) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc0024e0f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0019abb00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1111 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 1121 [select]: go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc00288a0a0, {0x6598810, 0xc002210150}) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:590 +0x1cf created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv in goroutine 1126 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:500 +0x2bb goroutine 1120 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1126 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 998 [syscall, 3774 minutes]: os/signal.signal_recv() /usr/local/go/src/runtime/sigqueue.go:152 +0x29 os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x13 created by os/signal.Notify.func1.1 in goroutine 1 /usr/local/go/src/os/signal/signal.go:151 +0x1f goroutine 997 [select, 2 minutes]: github.com/pingcap/tidb/disttask/framework/scheduler.(*Manager).recoverMetaLoop(0xc002228630) /go/src/github.com/pingcap/tidb/disttask/framework/scheduler/manager.go:275 +0xde github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 999 [chan receive, 3774 minutes]: github.com/pingcap/tidb/util/signal.SetupSignalHandler.func1() /go/src/github.com/pingcap/tidb/util/signal/signal_posix.go:37 +0x51 created by github.com/pingcap/tidb/util/signal.SetupSignalHandler in goroutine 1 /go/src/github.com/pingcap/tidb/util/signal/signal_posix.go:34 +0xa5 goroutine 995 [select]: github.com/pingcap/tidb/disttask/framework/scheduler.(*Manager).fetchAndHandleRunnableTasksLoop(0xc002228630) /go/src/github.com/pingcap/tidb/disttask/framework/scheduler/manager.go:146 +0xde github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 996 [select]: github.com/pingcap/tidb/disttask/framework/scheduler.(*Manager).fetchAndFastCancelTasksLoop(0xc002228630) /go/src/github.com/pingcap/tidb/disttask/framework/scheduler/manager.go:167 +0xde github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 1127 [select]: go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc00288a0a0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:550 +0x75 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 959 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:302 +0x96 goroutine 3105 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0410328, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc02e540080?, 0xc02e5c6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02e540080, {0xc02e5c6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc02e540080, {0xc02e5c6000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc02e5ba010, {0xc02e5c6000?, 0x0?, 0x5d6a070?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc02e5bc0c0, {0xc02eec9700, 0x9, 0x7f8b191aba68?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc02e5bc0c0}, {0xc02eec9700, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc02eec9700, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc02e5bc0c0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc02eec96c0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc02ddbdd40, 0xc02c05a7b8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4311 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 12511026 [sync.Cond.Wait, 1216 minutes]: sync.runtime_notifyListWait(0xc0f4953590, 0x20) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc0f94b7f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0f46d4b40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12510970 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 62888749 [select]: github.com/tiancaiamao/gp.workerLoop(0xc0010e0240) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:70 +0xb0 github.com/tiancaiamao/gp.worker(0xc0010e0240, 0xc353c69fa0?) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:51 +0x8d created by github.com/tiancaiamao/gp.(*Pool).Go in goroutine 56451313 /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:32 +0xb8 goroutine 930 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e06e0, {0x657b950, 0xc0021501e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 931 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e07c0, {0x657b950, 0xc002150460}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 932 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0820, {0x657b950, 0xc0021504b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 934 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0ac0, {0x657b950, 0xc002150870}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 935 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0c00, {0x657b950, 0xc002150af0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 936 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0c80, {0x657b950, 0xc002150b40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 108048 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c23bbe0, {0x657b950, 0xc005d68fa0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107998 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614514 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc195fc2fc0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614413 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 950 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001bad200) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 933 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 895 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 963 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 938 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e0fa0, {0x657b950, 0xc002150f00}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 939 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e10e0, {0x657b950, 0xc002151180}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 940 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1160, {0x657b950, 0xc0021511d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 942 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1440, {0x657b950, 0xc002151590}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 943 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e1580, {0x657b950, 0xc002151810}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 944 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0010e15e0, {0x657b950, 0xc002151860}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 912 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 962 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc00164c2a0, 0xc00151ea80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 912 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 963 [select]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc000b67400, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc000b67400, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 912 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 918 [select]: github.com/pingcap/tidb/domain.(*Domain).distTaskFrameworkLoop(0xc001736000, {0x657b918, 0xc0010fcf00}, 0xc000b34590, 0xc002228630, {0xc0012930c0, 0x31}) /go/src/github.com/pingcap/tidb/domain/domain.go:1492 +0x406 github.com/pingcap/tidb/domain.(*Domain).InitDistTaskLoop.func2() /go/src/github.com/pingcap/tidb/domain/domain.go:1451 +0x4c github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:96 +0x6c created by github.com/pingcap/tidb/util.(*WaitGroupEnhancedWrapper).Run in goroutine 1 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:91 +0xbd goroutine 896 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002429090, {0xc00142d210, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002429090, {0xc00142d210?, 0xc001351860?, 0xc00216b608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00142d1d0, {0xc00142d210?, 0xc00216b680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00142d1d0}, {0xc00142d210, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0022d2c60, {0xc00142d210, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00142d200, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc0022d2c60, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00142d200, {0x7f8ad0162008, 0x9529b80}, 0xc00216ba20?, {0x0, 0x0}, {0x598ae00, 0xc5ea617260}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bf040, {0x598ae00?, 0xc5ea617260}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc0022d2a20, 0xc00216baa8, 0xc00216ba98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc0022d2a20, {0x598ae00?, 0xc5ea617260?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc0000856b0) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc000e80fe0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc0021454a0, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 963 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 148780282 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1c618, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc598f50c00?, 0xc0bb7af000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc598f50c00, {0xc0bb7af000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc598f50c00, {0xc0bb7af000?, 0x1fb8c20?, 0x8c04de0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc44f6cbe08, {0xc0bb7af000?, 0x0?, 0xc00228b6c0?}) /usr/local/go/src/net/net.go:179 +0x45 net/http.(*persistConn).Read(0xc4087cb440, {0xc0bb7af000?, 0xc37741e540?, 0xc00f78dd38?}) /usr/local/go/src/net/http/transport.go:1954 +0x4a bufio.(*Reader).fill(0xc5a8e47680) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc5a8e47680, 0x1) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*persistConn).readLoop(0xc4087cb440) /usr/local/go/src/net/http/transport.go:2118 +0x1b9 created by net/http.(*Transport).dialConn in goroutine 148780276 /usr/local/go/src/net/http/transport.go:1776 +0x169f goroutine 951 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04472e8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc001a7e700?, 0xc002600000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc001a7e700, {0xc002600000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc001a7e700, {0xc002600000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001e74aa0, {0xc002600000?, 0xc002593f20?, 0xc001bc4cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0019c6f00, {0xc001f1bc40, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0019c6f00}, {0xc001f1bc40, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc001f1bc40, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0019c6f00?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc001f1bc00) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001bad200, 0xc000eb58c0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 933 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 919 [select]: github.com/pingcap/tidb/util/expensivequery.(*Handle).Run(0xc0011dd0b0) /go/src/github.com/pingcap/tidb/util/expensivequery/expensivequery.go:56 +0x169 created by main.createServer in goroutine 1 /go/src/github.com/pingcap/tidb/tidb-server/main.go:1187 +0x206 goroutine 921 [select]: github.com/pingcap/tidb/util/servermemorylimit.(*Handle).Run(0xc0011dd0f8) /go/src/github.com/pingcap/tidb/util/servermemorylimit/servermemorylimit.go:73 +0x165 created by main.createServer in goroutine 1 /go/src/github.com/pingcap/tidb/tidb-server/main.go:1189 +0x2c8 goroutine 968 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00139b680) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 945 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 969 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04476c8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc002251180?, 0xc002174000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc002251180, {0xc002174000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc002251180, {0xc002174000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc00198e330, {0xc002174000?, 0xc001432a80?, 0xc0024b4cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000b3c300, {0xc00212e2e0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000b3c300}, {0xc00212e2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc00212e2e0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc000b3c300?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00212e2a0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00139b680, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 945 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 980 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001b3f680) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 937 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 981 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04475d0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc000b67980?, 0xc0021ae000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000b67980, {0xc0021ae000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc000b67980, {0xc0021ae000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc001cd2208, {0xc0021ae000?, 0xc00125fe60?, 0xc001307cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc001300cc0, {0xc00219c040, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc001300cc0}, {0xc00219c040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc00219c040, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc001300cc0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00219c000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001b3f680, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 937 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 640 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000dc1270, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000a63880) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 941 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 970 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002151e00, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000da0e70) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 945 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 952 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0024d9090, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000dcbdc0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 933 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 982 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021983c0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000daa690) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 937 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 897 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 963 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 994 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002429270, {0xc00142d5d0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002429270, {0xc00142d5d0?, 0xc001351878?, 0xc002561608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00142d590, {0xc00142d5d0?, 0xc002561680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc00142d590}, {0xc00142d5d0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0022d2fc0, {0xc00142d5d0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc00142d5c0, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc0022d2fc0, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc00142d5c0, {0x7f8ad0162008, 0x9529b80}, 0xc002561a20?, {0x0, 0x0}, {0x598ae00, 0xc6012f9680}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bf1e0, {0x598ae00?, 0xc6012f9680}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc0022d2d80, 0xc002561aa8, 0xc002561a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc0022d2d80, {0x598ae00?, 0xc6012f9680?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc0001b40f0) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc000e81060) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc002145600, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 963 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 922 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 963 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 923 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001f5ad20, {0xc002204670, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001f5ad20, {0xc002204670?, 0xc000eab4b8?, 0xc00216d608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002204630, {0xc002204670?, 0xc00216d680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc002204630}, {0xc002204670, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001f49440, {0xc002204670, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc002204660, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc002204660?, 0xc001f49440, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc002204660, {0x7f8ad0162008, 0x9529b80}, 0xc00216da20?, {0x0, 0x0}, {0x598ae00, 0xc69b6475c0}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001f34b60, {0x598ae00?, 0xc69b6475c0}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc001f49200, 0xc00216daa8, 0xc00216da98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f49200, {0x598ae00?, 0xc69b6475c0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001583840) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc001ba5aa0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc002145760, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 963 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 983 [select]: github.com/tikv/pd/client/resource_group/controller.(*ResourceGroupsController).Start.func1() /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/resource_group/controller/controller.go:263 +0xbc5 created by github.com/tikv/pd/client/resource_group/controller.(*ResourceGroupsController).Start in goroutine 1 /go/cache/github.com/tikv/pd/client@v0.0.0-20240318054542-da3b6e381c9b/resource_group/controller/controller.go:211 +0xcd goroutine 20593059 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc3e8b3e460, {0xc3c483a940, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc3e8b3e460, {0xc3c483a940?, 0xc367668090?, 0xc001ea3608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc3c483a900, {0xc3c483a940?, 0xc001ea3680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc3c483a900}, {0xc3c483a940, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc4176fc240, {0xc3c483a940, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc3c483a930, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc4176fc240, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc3c483a930, {0x7f8ad0162008, 0x9529b80}, 0x175b0000175c?, {0x0, 0x0}, {0x598ae00, 0xc69b647980}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc365c04270, {0x598ae00?, 0xc69b647980}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc4176fc000, 0xc001ea3aa8, 0xc001ea3a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc4176fc000, {0x598ae00?, 0xc69b647980?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc44103e240) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc16e41c780) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc06da8eb00, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 20532494 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 1126 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00288c910, {0xc0023278a0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00288c910, {0xc0023278a0?, 0xc001351b90?, 0xc00255d9d8?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002327860, {0xc0023278a0?, 0xc00255da50?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc002327860}, {0xc0023278a0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028826c0, {0xc0023278a0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc002327890, 0x7fffffff) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x7f8a76449f88?, 0xc0028826c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc002327890, {0x7f8ad0162008, 0x9529b80}, 0x1c43665?, {0x0, 0x0}, {0x598e240, 0xc783d32f40}, 0x59b3040?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0028844e0, {0x598e240?, 0xc783d32f40}, 0x1c332da?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x295e125?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc002882480, 0xc00255de78, 0xc00255de68) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc002882480, {0x598e240?, 0xc783d32f40?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc002210150) /go/cache/go.etcd.io/etcd/api/v3@v3.5.10/etcdserverpb/rpc.pb.go:6853 +0x46 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc00288a0a0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:458 +0x258 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 in goroutine 959 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:301 +0x58 goroutine 1125 [select, 3774 minutes]: go.etcd.io/etcd/client/v3.(*lessor).keepAliveCtxCloser(0xc00288a0a0, {0x657b950, 0xc00217d4f0}, 0x1e65687?, 0xc0013028a0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:332 +0xc7 created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive in goroutine 959 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/lease.go:298 +0x5ce goroutine 1129 [select, 411 minutes]: github.com/pingcap/tidb/owner.(*ownerManager).watchOwner(0xc00288a140, {0x657b950?, 0xc002199c70?}, 0xc000b41920, {0xc051f4c6c0, 0x24}) /go/src/github.com/pingcap/tidb/owner/manager.go:436 +0x2e5 github.com/pingcap/tidb/owner.(*ownerManager).campaignLoop(0xc00288a140, 0xc000b41920) /go/src/github.com/pingcap/tidb/owner/manager.go:296 +0x839 created by github.com/pingcap/tidb/owner.(*ownerManager).CampaignOwner in goroutine 959 /go/src/github.com/pingcap/tidb/owner/manager.go:189 +0x2b9 goroutine 1131 [select, 2 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /go/cache/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 net/http.(*Server).Serve(0xc0017f9860, {0x656e410, 0xc001729cc0}) /usr/local/go/src/net/http/server.go:3056 +0x364 github.com/pingcap/tidb/server.(*Server).startStatusServerAndRPCServer.func2() /go/src/github.com/pingcap/tidb/server/http_status.go:525 +0x2b github.com/pingcap/tidb/util.WithRecovery(0x0?, 0x0?) /go/src/github.com/pingcap/tidb/util/misc.go:96 +0x48 created by github.com/pingcap/tidb/server.(*Server).startStatusServerAndRPCServer in goroutine 959 /go/src/github.com/pingcap/tidb/server/http_status.go:524 +0xe11 goroutine 164440301 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed79360, {0x657b950, 0xc6a3c004b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 1108 [select, 3774 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001a7a4c0, {0x657b950, 0xc00252b8b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 959 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6404031 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e46120, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc05d49d380?, 0xc006172000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05d49d380, {0xc006172000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc05d49d380, {0xc006172000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc05765a260, {0xc006172000?, 0x0?, 0xc58eda3c20?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0c9793b60, {0xc005d992a0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0c9793b60}, {0xc005d992a0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc005d992a0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0c9793b60?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc005d99260) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc025762000, 0xc0023bc7b8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403984 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 1239 [chan receive, 2877 minutes]: gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun(0xc001c4c120) /go/cache/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:379 +0x3f created by gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1 in goroutine 870 /go/cache/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:390 +0x96 goroutine 57614516 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc3b7798d70, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc197c79110) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614413 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 1025 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc0019c9750, 0x98a) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc002469f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001badb00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1112 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 12509516 [sync.Cond.Wait, 1225 minutes]: sync.runtime_notifyListWait(0xc0e707a010, 0x1f) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc0f5ef8f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0e6443440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 1130 [select, 3774 minutes]: github.com/soheilhy/cmux.muxListener.Accept(...) /go/cache/github.com/soheilhy/cmux@v0.1.5/cmux.go:262 google.golang.org/grpc.(*Server).Serve(0xc001fe5680, {0x656e410?, 0xc001729ce0}) /go/cache/google.golang.org/grpc@v1.59.0/server.go:852 +0x462 github.com/pingcap/tidb/server.(*Server).startStatusServerAndRPCServer.func1() /go/src/github.com/pingcap/tidb/server/http_status.go:520 +0x2b github.com/pingcap/tidb/util.WithRecovery(0x746c75616665645f?, 0x756e3a227469625f?) /go/src/github.com/pingcap/tidb/util/misc.go:96 +0x48 created by github.com/pingcap/tidb/server.(*Server).startStatusServerAndRPCServer in goroutine 959 /go/src/github.com/pingcap/tidb/server/http_status.go:519 +0xd53 goroutine 1277 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0410be0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0022de580?, 0xc02b0d0000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0022de580, {0xc02b0d0000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0022de580, {0xc02b0d0000?, 0x1fb8c20?, 0x8c04de0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000e962c8, {0xc02b0d0000?, 0x0?, 0xc00177e9c0?}) /usr/local/go/src/net/net.go:179 +0x45 net/http.(*persistConn).Read(0xc02947e120, {0xc02b0d0000?, 0xc002dc4a80?, 0xc0028ebd38?}) /usr/local/go/src/net/http/transport.go:1954 +0x4a bufio.(*Reader).fill(0xc02b0ce060) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc02b0ce060, 0x1) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*persistConn).readLoop(0xc02947e120) /usr/local/go/src/net/http/transport.go:2118 +0x1b9 created by net/http.(*Transport).dialConn in goroutine 1214 /usr/local/go/src/net/http/transport.go:1776 +0x169f goroutine 1119 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00288c6e0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000f213b0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1113 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 108031 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00af13f60, {0x657b950, 0xc005d513b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6404765 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0219e99a0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0085aaaf0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404633 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 1114 [select]: go.etcd.io/etcd/client/v3.(*Client).autoSync(0xc001767180) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:202 +0xb1 created by go.etcd.io/etcd/client/v3.newClient in goroutine 959 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/client.go:451 +0x110c goroutine 1118 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04109f0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc00222a880?, 0xc002890000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00222a880, {0xc002890000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc00222a880, {0xc002890000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0015852c0, {0xc002890000?, 0x0?, 0xc02376becc?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc000e32420, {0xc0022b79a0, 0x9, 0x7f8b191ac3c8?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc000e32420}, {0xc0022b79a0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0022b79a0, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc000e32420?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022b7960) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000ed9440, 0xc000eb58c0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1113 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164481231 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc58dcc3980, 0xc7204e3ec0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 164481194 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 322289 [select, 3598 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00b8ecc20, {0x657b950, 0xc0049649b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148779344 [select, 2 minutes]: github.com/pingcap/tidb/disttask/framework/dispatcher.(*Manager).cleanUpLoop(0xc6509fc750) /go/src/github.com/pingcap/tidb/disttask/framework/dispatcher/dispatcher_manager.go:297 +0x10b github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 5322553 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d6ab16d0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc05cfa75e0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322541 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164481282 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323cc20, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc477994000?, 0xc375162000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc477994000, {0xc375162000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc477994000, {0xc375162000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc6182494b8, {0xc375162000?, 0x8a4cd9f888?, 0xc6b668c1b0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc65a15c480, {0xc37337dd20, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc65a15c480}, {0xc37337dd20, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc37337dd20, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc65a15c480?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc37337dce0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc374c906c0, 0xc00212b7a0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481218 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 1190 [select, 3774 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 963 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 1191 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001725310, {0xc000e83030, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001725310, {0xc000e83030?, 0xc001cec660?, 0xc03e1ab608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000e82ff0, {0xc000e83030?, 0xc03e1ab680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc000e82ff0}, {0xc000e83030, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000f68480, {0xc000e83030, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc000e83020, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc000f68480, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc000e83020, {0x7f8ad0162008, 0x9529b80}, 0xc03e1aba20?, {0x0, 0x0}, {0x598ae00, 0xc69b6477a0}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc001773520, {0x598ae00?, 0xc69b6477a0}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc000f68240, 0xc03e1abaa8, 0xc03e1aba98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc000f68240, {0x598ae00?, 0xc69b6477a0?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc001181260) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc0023fa5c0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc002145340, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 963 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 164440216 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0dcaeb680) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440292 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 322308 [runnable]: github.com/tikv/client-go/v2/tikvrpc.keepOnlyActive({0xc6c1932000, 0x3904e1, 0x423400}, 0x17c37f9031a3f493) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1260 +0x38 github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc00442e480, 0xc000067f20) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1250 +0x25e created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 322228 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 6404030 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc025762000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403984 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164481221 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349c060, {0x657b950, 0xc6b9972190}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164499398 [select]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 164499396 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 11178096 [select, 1205 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0757ac140, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc03038fc00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178174 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 1278 [select]: net/http.(*persistConn).writeLoop(0xc02947e120) /usr/local/go/src/net/http/transport.go:2421 +0xe5 created by net/http.(*Transport).dialConn in goroutine 1214 /usr/local/go/src/net/http/transport.go:1777 +0x16f1 goroutine 4354 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02e5b2140, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc02e5de000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4311 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164481251 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad040ff48, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc5698e7080?, 0xc375040000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc5698e7080, {0xc375040000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc5698e7080, {0xc375040000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc623fda858, {0xc375040000?, 0x8a44cd6830?, 0xc5db37cc00?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc57efe4a20, {0xc36fab8900, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc57efe4a20}, {0xc36fab8900, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc36fab8900, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc57efe4a20?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc36fab88c0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc3726aab40, 0x0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481230 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 11178306 [select, 1209 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e6ebdf40, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc030fc51f0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178264 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164481198 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc374802d80) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481226 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164440241 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed787a0, {0x657b950, 0xc452164410}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11988920 [select, 3461 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0e951afe0, {0x657b950, 0xc0e9f3e320}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 12509518 [select, 1225 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e7000730, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0e694eee0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 163915771 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323ca30, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc6ae874680?, 0xc07f7e1000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc6ae874680, {0xc07f7e1000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc6ae874680, {0xc07f7e1000?, 0x1fb8c20?, 0x8c04de0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc46068e5e8, {0xc07f7e1000?, 0x0?, 0xc00228b6c0?}) /usr/local/go/src/net/net.go:179 +0x45 net/http.(*persistConn).Read(0xc66d68d0e0, {0xc07f7e1000?, 0xc69478ec60?, 0xc00e068d38?}) /usr/local/go/src/net/http/transport.go:1954 +0x4a bufio.(*Reader).fill(0xc50711bf80) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc50711bf80, 0x1) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*persistConn).readLoop(0xc66d68d0e0) /usr/local/go/src/net/http/transport.go:2118 +0x1b9 created by net/http.(*Transport).dialConn in goroutine 163916002 /usr/local/go/src/net/http/transport.go:1776 +0x169f goroutine 20532382 [select, 2349 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc139e981c0, {0x657b950, 0xc1b557a5a0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148780308 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e46028, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc65fde2300?, 0xc0bbbc9000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc65fde2300, {0xc0bbbc9000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc65fde2300, {0xc0bbbc9000?, 0x1fb8c20?, 0x8c04de0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc489b90d90, {0xc0bbbc9000?, 0x0?, 0xc00228b6c0?}) /usr/local/go/src/net/net.go:179 +0x45 net/http.(*persistConn).Read(0xc3d6ec7320, {0xc0bbbc9000?, 0xc37741ede0?, 0xc00f78bd38?}) /usr/local/go/src/net/http/transport.go:1954 +0x4a bufio.(*Reader).fill(0xc5263b31a0) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc5263b31a0, 0x1) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*persistConn).readLoop(0xc3d6ec7320) /usr/local/go/src/net/http/transport.go:2118 +0x1b9 created by net/http.(*Transport).dialConn in goroutine 148780291 /usr/local/go/src/net/http/transport.go:1776 +0x169f goroutine 148780006 [chan receive, 411 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 787 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 57614273 [select, 2349 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 20593064 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 6404742 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323c558, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc060d0a700?, 0xc0090d4000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc060d0a700, {0xc0090d4000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc060d0a700, {0xc0090d4000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0c0e18c70, {0xc0090d4000?, 0x0?, 0xc6cc0e8f30?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc085c7cd80, {0xc061dd8e40, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc085c7cd80}, {0xc061dd8e40, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc061dd8e40, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc085c7cd80?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc061dd8e00) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0d2700240, 0xc00c9c7fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404682 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164481283 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc6db1ae370, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc373a45f80) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481218 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 5323049 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323c748, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc089076c00?, 0xc060da6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc089076c00, {0xc060da6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc089076c00, {0xc060da6000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc062126a68, {0xc060da6000?, 0xc3ed608750?, 0xc70844d3e0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0c4589560, {0xc05ff97380, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0c4589560}, {0xc05ff97380, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc05ff97380, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0c4589560?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc05ff97340) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc060a66000, 0xc002363fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323020 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 20593045 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc37757af00, {0xc4039d30f0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc37757af00, {0xc4039d30f0?, 0xc3384f5b48?, 0xc0010c7608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc4039d30b0, {0xc4039d30f0?, 0xc0010c7680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc4039d30b0}, {0xc4039d30f0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc3ff166d80, {0xc4039d30f0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc4039d30e0, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc4039d30e0?, 0xc3ff166d80, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc4039d30e0, {0x7f8ad0162008, 0x9529b80}, 0xc0010c7a20?, {0x0, 0x0}, {0x598ae00, 0xc65390c360}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc3beff0dd0, {0x598ae00?, 0xc65390c360}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc3ff166a20, 0xc0010c7aa8, 0xc0010c7a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc3ff166a20, {0x598ae00?, 0xc65390c360?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc42c922340) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc144e9a8e0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc06da8ec60, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 20532494 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 148780018 [chan receive, 411 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 725 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 3104 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc02ddbdd40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4311 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 12509980 [sync.Cond.Wait, 1213 minutes]: sync.runtime_notifyListWait(0xc0ec4841d0, 0x20) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc093baaf70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0ec226240) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509997 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 5322057 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0587e7b00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322012 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 6403460 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0a14686c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403329 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164481168 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc5fea4d260, {0x657b950, 0xc6360b3db0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481224 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349c660, {0x657b950, 0xc7235c8280}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 4289 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8ad0410420, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc02f4ac100?, 0xc02f3f2000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02f4ac100, {0xc02f3f2000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc02f4ac100, {0xc02f3f2000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc02f0e2318, {0xc02f3f2000?, 0x5716260?, 0xc6fc68de30?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc02f3bc600, {0xc02f0e6d60, 0x9, 0x7f8b191aba68?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc02f3bc600}, {0xc02f0e6d60, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc02f0e6d60, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc02f3bc600?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc02f0e6d20) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc02e3958c0, 0xc02f0e22f0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4309 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 322201 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009edffc0, {0x657b950, 0xc003eccf00}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164499404 [select]: github.com/tikv/client-go/v2/txnkv/transaction.keepAlive(0xc6aaea68c0, 0xc6a75a33e0, {0xc64c9694d0, 0x13, 0x13}, 0xc6aacb15e0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:1181 +0x175 created by github.com/tikv/client-go/v2/txnkv/transaction.(*ttlManager).run in goroutine 129173895 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:1150 +0x1c6 goroutine 322237 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7f60, {0x657b950, 0xc004b29ea0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614301 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc192a67440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614268 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 12509981 [IO wait, 1213 minutes]: internal/poll.runtime_pollWait(0x7f8ad040fe50, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0ebd11f80?, 0xc0ec4b0000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0ebd11f80, {0xc0ec4b0000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0ebd11f80, {0xc0ec4b0000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0e83d1f58, {0xc0ec4b0000?, 0x8a59702960?, 0xc69420f590?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0ec21b560, {0xc0eb9f6e40, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0ec21b560}, {0xc0eb9f6e40, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0eb9f6e40, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0ec21b560?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0eb9f6e00) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0ec226240, 0xc00e447fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509997 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 108033 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c289ae0, {0x657b950, 0xc005d51680}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 12509693 [IO wait, 1213 minutes]: internal/poll.runtime_pollWait(0x7f8ad0410138, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0eab2a300?, 0xc0eb060000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0eab2a300, {0xc0eb060000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0eab2a300, {0xc0eb060000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0e83d0800, {0xc0eb060000?, 0x8a4d820158?, 0xc692ec0ab0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0eac78d80, {0xc0ea7b2660, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0eac78d80}, {0xc0ea7b2660, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0ea7b2660, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0eac78d80?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0ea7b2620) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0e8c89d40, 0xc001bf6fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509848 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 322239 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009bf4460, {0x657b950, 0xc00505c190}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532485 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0d5b9d320, {0x657b950, 0xc16d56ac30}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322229 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7880, {0x657b950, 0xc004b29180}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614302 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1cdd8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc34c198780?, 0xc195f40000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc34c198780, {0xc195f40000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc34c198780, {0xc195f40000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc439bd40d0, {0xc195f40000?, 0xc3289c7980?, 0xc00d519cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc3bb5c8780, {0xc19461a3c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc3bb5c8780}, {0xc19461a3c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc19461a3c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc3bb5c8780?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc19461a380) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc192a67440, 0xc00d519fb8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614268 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 148758147 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e1cce0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc47dea1300?, 0xc004ec2000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc47dea1300, {0xc004ec2000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc47dea1300, {0xc004ec2000?, 0xc24e4290c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc1c7c4d0e0, {0xc004ec2000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5284f6f60, {0xc687cd8a98, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc5e4e7a060}, {0xc687cd8a98, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc44b576a20) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc44b576a20) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc00af7b860) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc00af7b860, {0x657b918, 0xc4501340c0}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc00af7b860) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 322206 [select, 3619 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21ec00, {0x657b950, 0xc003ecd810}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11178094 [sync.Cond.Wait, 1205 minutes]: sync.runtime_notifyListWait(0xc0bc492090, 0x24) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc0023d4f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0302f1b00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178174 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164440287 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1c428, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc67e94dd00?, 0xc0df698000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc67e94dd00, {0xc0df698000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc67e94dd00, {0xc0df698000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc6507a4910, {0xc0df698000?, 0x20?, 0xc798d0bd40?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc6d56d68a0, {0xc0defca3c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc6d56d68a0}, {0xc0defca3c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0defca3c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc6d56d68a0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0defca380) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0deac8900, 0xc0d946001c?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440296 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 4286 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02f3ba870, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc02f1eb3b0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4310 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 11174009 [select, 3479 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0e887f020, {0x657b950, 0xc0b4111680}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 2467 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0447000, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc02c892000?, 0xc02c8a6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02c892000, {0xc02c8a6000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc02c892000, {0xc02c8a6000?, 0x0?, 0x0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc02c896000, {0xc02c8a6000?, 0xc04244da48?, 0x1d2dfc5?}) /usr/local/go/src/net/net.go:179 +0x45 github.com/soheilhy/cmux.(*bufferedReader).Read(0xc02c898010, {0xc02c8a6000, 0x0?, 0x1000}) /go/cache/github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f github.com/soheilhy/cmux.(*MuxConn).Read(0xc02c8940c0?, {0xc02c8a6000?, 0xc04244daf8?, 0x0?}) /go/cache/github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e net/http.(*connReader).Read(0xc02c8940c0, {0xc02c8a6000, 0x1000, 0x1000}) /usr/local/go/src/net/http/server.go:791 +0x14b bufio.(*Reader).fill(0xc02c779da0) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc02c779da0, 0x4) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*conn).serve(0xc02c8a2000, {0x657b918, 0xc000b41b90}) /usr/local/go/src/net/http/server.go:2044 +0x75c created by net/http.(*Server).Serve in goroutine 1131 /usr/local/go/src/net/http/server.go:3086 +0x5cb goroutine 148780283 [select]: net/http.(*persistConn).writeLoop(0xc4087cb440) /usr/local/go/src/net/http/transport.go:2421 +0xe5 created by net/http.(*Transport).dialConn in goroutine 148780276 /usr/local/go/src/net/http/transport.go:1777 +0x16f1 goroutine 322203 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21e900, {0x657b950, 0xc003ecd1d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 5322058 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323cf08, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0e4cb4a80?, 0xc058df6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0e4cb4a80, {0xc058df6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0e4cb4a80, {0xc058df6000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0d74939a0, {0xc058df6000?, 0x10?, 0xc60c825e00?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc09c016060, {0xc058dea040, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc09c016060}, {0xc058dea040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc058dea040, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc09c016060?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc058dea000) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0587e7b00, 0xc00e481fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322012 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 4288 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc02f3b8690, 0x2296) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc02c05a770?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc02e3958c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4309 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164481250 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc3726aab40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481230 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164499397 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc6c0506ea0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:327 +0x7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:342 google.golang.org/grpc.(*csAttempt).recvMsg(0xc786f07450, {0x59cbf40?, 0xc7842ae380}, 0xc653cb7ee8?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1070 +0xba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc6c0506c60, 0xc24e42cac0, 0xc24e42cab0) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:776 +0x13a google.golang.org/grpc.(*clientStream).RecvMsg(0xc6c0506c60, {0x59cbf40?, 0xc7842ae380?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvEstablishMPPConnectionClient).Recv(0xc537964bc0) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2185 +0x46 github.com/tikv/client-go/v2/tikvrpc.(*MPPStreamResponse).Recv(0xc70844d950) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1209 +0x98 github.com/tikv/client-go/v2/internal/client.(*RPCClient).getMPPStreamResponse(0xc000f268c0?, {0x657b950?, 0xc5fc475db0?}, {0x6604ff0, 0xc5b59335d8}, 0x0?, 0x34630b8a000, 0xc009f5faa0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:822 +0xf9 github.com/tikv/client-go/v2/internal/client.(*RPCClient).sendRequest(0xc000f268c0, {0x657b950, 0xc5fc475db0}, {0xc6a8990c30, 0x4b}, 0xc6aa37d380, 0x657b918?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:703 +0xb4d github.com/tikv/client-go/v2/internal/client.(*RPCClient).SendRequest(0xc000f268c0, {0x657b950, 0xc5fc475db0}, {0xc6a8990c30, 0x4b}, 0x0?, 0xc423a2ed40?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:723 +0xaf github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest({{0x6565730?, 0xc000f268c0?}}, {0x657b950, 0xc5fc475db0}, {0xc6a8990c30, 0x4b}, 0xc02c260680?, 0x34630b8a000) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_interceptor.go:60 +0x210 github.com/tikv/client-go/v2/internal/client.reqCollapse.SendRequest({{0x6565a60?, 0xc000f1eea0?}}, {0x657b950, 0xc5fc475db0}, {0xc6a8990c30, 0x4b}, 0xc04244f250?, 0xc24e42d270?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_collapse.go:74 +0xc3 github.com/pingcap/tidb/store/copr.(*tikvClient).SendRequest(0x0?, {0x657b950?, 0xc5fc475db0?}, {0xc6a8990c30?, 0xc5f182ff08?}, 0x13?, 0x657b9c0?) /go/src/github.com/pingcap/tidb/store/copr/store.go:72 +0x2a github.com/pingcap/tidb/store/copr.(*mppIterator).establishMPPConns(0xc6a9c77f40, 0xc5b5933070, 0xc6585d81b0, 0xc6aa81fc80) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:443 +0x305 github.com/pingcap/tidb/store/copr.(*mppIterator).handleDispatchReq(0xc6a9c77f40, {0x657b950, 0xc5fc475db0}, 0xc5b5933070?, 0xc6585d81b0) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:360 +0x19f7 github.com/pingcap/tidb/store/copr.(*mppIterator).run.func2(0xc00c9c5fa0?) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:194 +0x5b created by github.com/pingcap/tidb/store/copr.(*mppIterator).run in goroutine 164499410 /go/src/github.com/pingcap/tidb/store/copr/mpp.go:190 +0x4f goroutine 148779342 [select]: github.com/pingcap/tidb/disttask/framework/dispatcher.(*Manager).dispatchTaskLoop(0xc6509fc750) /go/src/github.com/pingcap/tidb/disttask/framework/dispatcher/dispatcher_manager.go:160 +0x165 github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1() /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154 +0x4f created by github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run in goroutine 918 /go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:152 +0x73 goroutine 12509982 [select, 1213 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0ec237220, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0ebd9cd90) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509997 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 322197 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009edfc40, {0x657b950, 0xc003ecc870}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614503 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc196012240) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614409 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 20593064 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00ac0b3b0, {0xc387e7afd0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00ac0b3b0, {0xc387e7afd0?, 0xc3b8386078?, 0xc001d0d608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc387e7af90, {0xc387e7afd0?, 0xc001d0d680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc387e7af90}, {0xc387e7afd0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc3cc2e2b40, {0xc387e7afd0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc387e7afc0, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc3cc2e2b40, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc387e7afc0, {0x7f8ad0162008, 0x9529b80}, 0xc001d0da20?, {0x0, 0x0}, {0x598ae00, 0xc6d59d5800}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc3f98c01a0, {0x598ae00?, 0xc6d59d5800}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc3cc2e25a0, 0xc001d0daa8, 0xc001d0da98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc3cc2e25a0, {0x598ae00?, 0xc6d59d5800?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc43bb005c0) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc1c08c4ba0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc06da8e840, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 20532494 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 20592941 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc3b7799130, {0xc4245d1a80, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc3b7799130, {0xc4245d1a80?, 0xc432f58450?, 0xc0010c9608?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc4245d1a40, {0xc4245d1a80?, 0xc0010c9680?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc4245d1a40}, {0xc4245d1a80, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc3938d3c20, {0xc4245d1a80, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc4245d1a70, 0x7ffffffffffffffe) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0x9529b80?, 0xc3938d3c20, {0x0, 0x0}, 0x7ffffffffffffffe, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc4245d1a70, {0x7f8ad0162008, 0x9529b80}, 0xc0010c9a20?, {0x0, 0x0}, {0x598ae00, 0xc6d59d5c20}, 0x1d22306?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc3ee0804e0, {0x598ae00?, 0xc6d59d5c20}, 0x1c43665?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc3938d2a20, 0xc0010c9aa8, 0xc0010c9a98) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc3938d2a20, {0x598ae00?, 0xc6d59d5c20?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvBatchCommandsClient).Recv(0xc431468c20) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2135 +0x46 github.com/tikv/client-go/v2/internal/client.(*batchCommandsStream).recv(0xc1b837bca0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:458 +0x182 github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).batchRecvLoop(0xc06da8e9a0, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}, ...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:621 +0xc9 created by github.com/tikv/client-go/v2/internal/client.(*batchCommandsClient).initBatchClient in goroutine 20532494 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:756 +0x1f5 goroutine 322202 [select, 3619 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21e8a0, {0x657b950, 0xc003ecd180}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481267 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc37419d200) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481222 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 6404763 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc036764d80) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404633 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 4370 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02f3bab90, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc02f270460) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4309 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164439969 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc4130940f0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0db87ae00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440300 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 4285 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0410230, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc02f344080?, 0xc02f3d6000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02f344080, {0xc02f3d6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc02f344080, {0xc02f3d6000?, 0x1060100000000?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc02f0e22f0, {0xc02f3d6000?, 0x0?, 0xc042eabecc?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc02f3bc420, {0xc02f0e6c80, 0x9, 0x7f8b191ac3c8?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc02f3bc420}, {0xc02f0e6c80, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc02f0e6c80, 0x9, 0x94f6a80?}, {0x65496a0?, 0xc02f3bc420?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc02f0e6c40) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc02e395440, 0xc02ba6001c?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 4310 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 148760365 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc6542deea0, 0xc00ea3db80, 0xc3c59b03c0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 148760361 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 6403462 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f0a63ef0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc04160d6c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403329 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 322235 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7d40, {0x657b950, 0xc004b29ae0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440299 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed790c0, {0x657b950, 0xc61e49b220}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6404741 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0d2700240) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404682 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 11178598 [sync.Cond.Wait, 1205 minutes]: sync.runtime_notifyListWait(0xc0ab8f6510, 0x24) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc02d60bf70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc032175440) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178588 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 7457103 [select, 3479 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc05ac06720, {0x657b950, 0xc076043400}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148760363 [select, 411 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 148760361 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 7457104 [select, 3568 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc05ac06a00, {0x657b950, 0xc0760434a0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 5323984 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc06a5c9200) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323887 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 57614517 [select, 2349 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 20592941 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 322355 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21efc0, {0x657b950, 0xc003ecdef0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481200 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc6cb4c3680, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc374aa02a0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481226 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 57614303 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc3c1266aa0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc193458af0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614268 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 11178389 [select, 1218 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0b3c37c70, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0314ac930) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178359 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 163915772 [select]: net/http.(*persistConn).writeLoop(0xc66d68d0e0) /usr/local/go/src/net/http/transport.go:2421 +0xe5 created by net/http.(*Transport).dialConn in goroutine 163916002 /usr/local/go/src/net/http/transport.go:1777 +0x16f1 goroutine 7273325 [select, 3479 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc033353ac0, {0x657b950, 0xc0934a0b90}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532494 [select]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc1c79df000, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc1c79df000, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 20532273 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 322357 [runnable]: github.com/tikv/client-go/v2/tikvrpc.keepOnlyActive({0xc78e576000, 0x8f305, 0xb1400}, 0x17c37f907977dcb0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1260 +0x3c github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc009f5fa40, 0xc00b615c20) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1250 +0x25e created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 322255 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 322241 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009bf4680, {0x657b950, 0xc00505c5a0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6403461 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1cbe8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc02684df00?, 0xc003a46000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02684df00, {0xc003a46000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc02684df00, {0xc003a46000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc035e7d988, {0xc003a46000?, 0x2?, 0xc730cf3320?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc03c072f00, {0xc0a0b043c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc03c072f00}, {0xc0a0b043c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0a0b043c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc03c072f00?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a0b04380) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0a14686c0, 0xc05bd37220?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403329 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164440294 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed78c80, {0x657b950, 0xc452165b30}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322198 [select, 3619 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009edfd20, {0x657b950, 0xc003eccaf0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440293 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed78ba0, {0x657b950, 0xc4521653b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 108023 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00af138a0, {0x657b950, 0xc005d50690}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614393 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc1959a5200) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614436 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 108032 [select, 3479 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c289a80, {0x657b950, 0xc005d51630}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614505 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc38f902190, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19722b180) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614409 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164440298 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed79060, {0x657b950, 0xc61e49aff0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532490 [select, 2349 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc137a1f0a0, {0x657b950, 0xc1b6c14910}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614395 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc39a2a9590, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19753a9a0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614436 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164499399 [select]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 164499397 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 148760364 [select, 411 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc38e87d270, {0xc3f131ded0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:191 +0x98 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc38e87d270, {0xc3f131ded0?, 0xc2d028b050?, 0xc03d6959f0?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:171 +0x12d google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc41a8fc000, {0xc3f131ded0?, 0xc03d695a68?, 0x21be385?}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:506 +0x2c io.ReadAtLeast({0x654eec0, 0xc41a8fc000}, {0xc3f131ded0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc44c2638c0, {0xc3f131ded0, 0x5, 0x5}) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:490 +0x96 google.golang.org/grpc.(*parser).recvMsg(0xc3f131dec0, 0x7fffffff) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:600 +0x46 google.golang.org/grpc.recvAndDecompress(0xc03d695c48?, 0xc44c2638c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:730 +0x5a google.golang.org/grpc.recv(0xc3f131dec0, {0x7f8ad0162008, 0x9529b80}, 0x0?, {0x0, 0x0}, {0x59fbf00, 0xc0314ad960}, 0xc001a534f0?, 0x0, ...) /go/cache/google.golang.org/grpc@v1.59.0/rpc_util.go:796 +0xa6 google.golang.org/grpc.(*csAttempt).recvMsg(0xc013004dd0, {0x59fbf00?, 0xc0314ad960}, 0xc03d695de0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1084 +0x27d google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc03d695de0?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc44c263680, 0xc03d695e90, 0xc03d695e80) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:760 +0x3ae google.golang.org/grpc.(*clientStream).RecvMsg(0xc44c263680, {0x59fbf00?, 0xc0314ad960?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc4c7c8bb90) /go/cache/go.etcd.io/etcd/api/v3@v3.5.10/etcdserverpb/rpc.pb.go:6714 +0x46 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc6542deea0, {0x65988c0, 0xc4c7c8bb90}) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:766 +0x5c created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 148760361 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:921 +0x598 goroutine 322199 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009edfd80, {0x657b950, 0xc003eccb40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148760366 [chan receive, 411 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 1129 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 108027 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00af13c00, {0x657b950, 0xc005d50d20}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164499410 [semacquire]: sync.runtime_Semacquire(0xc6fb7f94b8?) /usr/local/go/src/runtime/sema.go:62 +0x25 sync.(*WaitGroup).Wait(0x657b918?) /usr/local/go/src/sync/waitgroup.go:116 +0x48 github.com/pingcap/tidb/store/copr.(*mppIterator).run(0xc6a9c77f40, {0x657b950?, 0xc5fc475db0}) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:197 +0x2d1 created by github.com/pingcap/tidb/store/copr.(*MPPClient).DispatchMPPTasks in goroutine 129173939 /go/src/github.com/pingcap/tidb/store/copr/mpp.go:599 +0x2f6 goroutine 7273326 [select, 3577 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc033353d60, {0x657b950, 0xc0934a0dc0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 108040 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c23b520, {0x657b950, 0xc005d68280}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107998 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11178599 [IO wait, 1205 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e1c710, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0f7f3a180?, 0xc0319ec000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0f7f3a180, {0xc0319ec000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0f7f3a180, {0xc0319ec000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc07852a0d8, {0xc0319ec000?, 0x8a6e8396a8?, 0xc61f83fc20?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc07ceaaa80, {0xc031f12f20, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc07ceaaa80}, {0xc031f12f20, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc031f12f20, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc07ceaaa80?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc031f12ee0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc032175440, 0xc02d60dfa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178588 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 5323050 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0c69c65a0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc060a58ee0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323020 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164481199 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323c650, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc6a1c5d980?, 0xc374f1c000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc6a1c5d980, {0xc374f1c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc6a1c5d980, {0xc374f1c000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc6a9711b88, {0xc374f1c000?, 0x8a4266a060?, 0xc5e06a3c50?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc6541fac60, {0xc373f78ba0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc6541fac60}, {0xc373f78ba0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc373f78ba0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc6541fac60?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc373f78b60) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc374802d80, 0x2?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481226 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 11988919 [select, 3448 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0e951af20, {0x657b950, 0xc0e9f3e230}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614515 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad04108f8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc4492b7f00?, 0xc19975a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc4492b7f00, {0xc19975a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc4492b7f00, {0xc19975a000?, 0x100000001?, 0xc019117c90?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc36869e4a8, {0xc19975a000?, 0xc3505940c0?, 0xc019117cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc4075ebf80, {0xc19604ee40, 0x9, 0x10000001a?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc4075ebf80}, {0xc19604ee40, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc19604ee40, 0x9, 0xc4245d1710?}, {0x65496a0?, 0xc4075ebf80?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc19604ee00) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc195fc2fc0, 0xc019117fb8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614413 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 108044 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c23b880, {0x657b950, 0xc005d68910}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107998 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 108035 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c289d00, {0x657b950, 0xc005d51a40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107997 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6404032 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc03080b0e0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc02c899c00) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6403984 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 57614394 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0479f60, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc3d1e2fb00?, 0xc1987c4000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc3d1e2fb00, {0xc1987c4000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc3d1e2fb00, {0xc1987c4000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc3b889b190, {0xc1987c4000?, 0xc3fecbafc0?, 0xc002540cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc42ee8cc60, {0xc196a411c0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc42ee8cc60}, {0xc196a411c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc196a411c0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc42ee8cc60?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc196a41180) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc1959a5200, 0xc002540fb8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614436 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 129173895 [select]: github.com/tikv/client-go/v2/internal/client.sendBatchRequest({0x657b918?, 0xc441d1f860}, {0xc02dec2cc0, 0x32}, {0x0, 0x0}, 0xc1c79df000, 0xc6de3b9110, 0x3f5c6616792d015a?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:807 +0x930 github.com/tikv/client-go/v2/internal/client.(*RPCClient).sendRequest(0xc000f268c0, {0x657b918, 0xc441d1f860}, {0xc02dec2cc0, 0x32}, 0xc6aadcfa00, 0x7?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:674 +0x626 github.com/tikv/client-go/v2/internal/client.(*RPCClient).SendRequest(0xc000f268c0, {0x657b918, 0xc441d1f860}, {0xc02dec2cc0, 0x32}, 0xc039bdc418?, 0xc039bdc478?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:723 +0xaf github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest.func1({0xc02dec2cc0?, 0xc042c7f8c0?}, 0x657b918?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_interceptor.go:57 +0x3f github.com/pingcap/tidb/session.(*session).doCommit.(*KvExecCounter).RPCInterceptor.func4.1({0xc02dec2cc0, 0x32}, 0xc441d1f860?) /go/src/github.com/pingcap/tidb/util/topsql/stmtstats/kv_exec_count.go:57 +0x65 github.com/tikv/client-go/v2/internal/client.buildResourceControlInterceptor.func1.1({0xc02dec2cc0, 0x32}, 0xc6aadcf860) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_interceptor.go:124 +0x1cd github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest({{0x6565730?, 0xc000f268c0?}}, {0x657b918, 0xc441d1f860}, {0xc02dec2cc0, 0x32}, 0xc0010e2420?, 0x6fc23ac00) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_interceptor.go:58 +0x1cd github.com/tikv/client-go/v2/internal/client.reqCollapse.SendRequest({{0x6565a60?, 0xc000f1eea0?}}, {0x657b918, 0xc441d1f860}, {0xc02dec2cc0, 0x32}, 0xc039bdc6c0?, 0x1cd56cf?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_collapse.go:74 +0xc3 github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).sendReqToRegion(0xc6f116b560, 0xc650240480, 0xc650240510, 0xc6aadcf860, 0xc039bdcf18?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_request.go:1712 +0x8c9 github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).SendReqCtx(0xc6f116b560, 0xc650240480, 0xc6aadcf860, {0x116939, 0x25, 0x78}, 0x6fc23ac00, 0x0, {0x0, 0x0, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_request.go:1470 +0x14be github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).SendReq(...) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/locate/region_request.go:252 github.com/tikv/client-go/v2/txnkv/transaction.actionCommit.handleSingleBatch({0x36?, 0x0?}, 0xc6aaea68c0, 0xc650240480, {{0x116939, 0x25, 0x78}, {0x659c930, 0xc6856a9e60}, 0x1}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/commit.go:107 +0x805 github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).doActionOnBatches(0xc6aaea68c0, 0xc650240480, {0x6565b50?, 0xc44d687378}, {0xc6f116b500?, 0x1, 0xc039bddd40?}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:1084 +0x54f github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).doActionOnGroupMutations(0xc6aaea68c0, 0xc650240480, {0x6565b50, 0xc44d687378?}, {0xc6bc844d20, 0x2, 0xc039bde000?}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:989 +0x9a5 github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).doActionOnMutations(0xc6aaea68c0, 0xc441d1f860?, {0x6565b50?, 0xc44d687378}, {0x659c930, 0xc6e5c61e80}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:808 +0xc5 github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).commitMutations(0xc6aaea68c0, 0xc650240480, {0x659c930, 0xc6e5c61e80}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/commit.go:231 +0x25a github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).commitTxn(0xc6aaea68c0, {0x657b918?, 0xc441d1f860}, 0xc69a801440) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:1727 +0x1a8 github.com/tikv/client-go/v2/txnkv/transaction.(*twoPhaseCommitter).execute(0xc6aaea68c0, {0x657b918?, 0xc441d1f860}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/2pc.go:1718 +0x233f github.com/tikv/client-go/v2/txnkv/transaction.(*KVTxn).Commit(0xc6aae9d080, {0x657b918, 0xc62e05fec0?}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/txnkv/transaction/txn.go:508 +0xf4b github.com/pingcap/tidb/store/driver/txn.(*tikvTxn).Commit(0xc408ead8c0, {0x657b918?, 0xc62e05fec0?}) /go/src/github.com/pingcap/tidb/store/driver/txn/txn_driver.go:99 +0x28 github.com/pingcap/tidb/session.(*LazyTxn).Commit(0xc45041ac90, {0x657b918, 0xc62e05fec0}) /go/src/github.com/pingcap/tidb/session/txn.go:429 +0x51f github.com/pingcap/tidb/session.(*session).commitTxnWithTemporaryData(0x657b918?, {0x657b918?, 0xc62e05fec0?}, {0x65cb0c0?, 0xc45041ac90?}) /go/src/github.com/pingcap/tidb/session/session.go:809 +0x1b5 github.com/pingcap/tidb/session.(*session).doCommit(0xc45041ac80, {0x657b918, 0xc62e05fda0}) /go/src/github.com/pingcap/tidb/session/session.go:690 +0xdeb github.com/pingcap/tidb/session.(*session).doCommitWithRetry(0xc45041ac80, {0x657b918, 0xc62e05fda0}) /go/src/github.com/pingcap/tidb/session/session.go:943 +0x206 github.com/pingcap/tidb/session.(*session).CommitTxn(0xc45041ac80, {0x657b918?, 0xc62e05f860?}) /go/src/github.com/pingcap/tidb/session/session.go:1070 +0xf8 github.com/pingcap/tidb/session.autoCommitAfterStmt({0x657b918, 0xc62e05f860}, 0xc45041ac80, {0x0?, 0x0}, {0x658dde0?, 0xc6ab0d43c0}) /go/src/github.com/pingcap/tidb/session/tidb.go:297 +0x465 github.com/pingcap/tidb/session.finishStmt({0x657b918, 0xc62e05f860}, 0xc45041ac80, {0x0, 0x0}, {0x658dde0, 0xc6ab0d43c0?}) /go/src/github.com/pingcap/tidb/session/tidb.go:259 +0x279 github.com/pingcap/tidb/session.runStmt({0x657b918?, 0xc62e05f860?}, 0xc45041ac80, {0x658dde0, 0xc6ab0d43c0?}) /go/src/github.com/pingcap/tidb/session/session.go:2432 +0x7c5 github.com/pingcap/tidb/session.(*session).ExecuteStmt(0xc45041ac80, {0x657b918?, 0xc62e05f860?}, {0x6593fb8?, 0xc6ab0c1220?}) /go/src/github.com/pingcap/tidb/session/session.go:2262 +0x107b github.com/pingcap/tidb/server.(*TiDBContext).ExecuteStmt(0xc489d06d68, {0x657b918, 0xc62e05f860}, {0x6593fb8, 0xc6ab0c1220?}) /go/src/github.com/pingcap/tidb/server/driver_tidb.go:315 +0x20a github.com/pingcap/tidb/server.(*clientConn).executePreparedStmtAndWriteResult(0xc44f9c3040, {0x657b918, 0xc62e05f860}, {0x65a8ba0, 0xc38faf6c00?}, {0x9529b80, 0x0, 0x0}, 0x0) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:308 +0x5b3 github.com/pingcap/tidb/server.(*clientConn).executePlanCacheStmt(0xc44f9c3040, {0x657b950, 0xc5d0632a50}, {0x59396c0?, 0xc38faf6c00}, {0x9529b80, 0x0, 0x0}, 0x52?) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:236 +0x185 github.com/pingcap/tidb/server.(*clientConn).handleStmtExecute(0xc44f9c3040, {0x657b950, 0xc5d0632a50}, {0xc55c7d7be5, 0x9, 0x9}) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:227 +0x8dd github.com/pingcap/tidb/server.(*clientConn).dispatch(0xc44f9c3040, {0x657b918?, 0xc5c7991020?}, {0xc55c7d7be4, 0xa, 0xa}) /go/src/github.com/pingcap/tidb/server/conn.go:1483 +0x139e github.com/pingcap/tidb/server.(*clientConn).Run(0xc44f9c3040, {0x657b918, 0xc5c7991020}) /go/src/github.com/pingcap/tidb/server/conn.go:1234 +0x294 github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc44f9c3040) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 108072 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc005d49800, 0xc0058ebec0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 107998 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 12511028 [select, 1216 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f4da0550, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0f494eb60) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12510970 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 7013961 [select, 3479 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0ea5e8fa0, {0x657b950, 0xc01f5e50e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322233 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7c00, {0x657b950, 0xc004b29810}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148758145 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8aa323d1f0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc2b964df80?, 0xc004e2a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc2b964df80, {0xc004e2a000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc2b964df80, {0xc004e2a000?, 0xc24e2230c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc1bd7b9d58, {0xc004e2a000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5284f6c60, {0xc2dc7b8fb8, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc5627bdcf8}, {0xc2dc7b8fb8, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc44b5768a0) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc44b5768a0) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc00af7b520) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc00af7b520, {0x657b918, 0xc44e2000c0}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc00af7b520) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 322234 [select, 3598 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7ce0, {0x657b950, 0xc004b29a90}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11408089 [select, 3448 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0effa2300, {0x657b950, 0xc06ca6eaf0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481153 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc374c906c0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481218 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 148760361 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc6542deea0) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:545 +0x33b created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 1129 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:292 +0x2ff goroutine 322207 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21ec60, {0x657b950, 0xc003ecd860}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 108068 [select, 3756 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c23bf40, {0x657b950, 0xc005d69630}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 107998 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20593234 [select]: github.com/tiancaiamao/gp.workerLoop(0xc0010e0240) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:70 +0xb0 github.com/tiancaiamao/gp.worker(0xc0010e0240, 0xc00390d7a0?) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:51 +0x8d created by github.com/tiancaiamao/gp.(*Pool).Go in goroutine 20593043 /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:32 +0xb8 goroutine 20532489 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc137a1ec60, {0x657b950, 0xc1b6c140f0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 5323985 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323d2e8, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0d14ffa00?, 0xc06ae9a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0d14ffa00, {0xc06ae9a000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0d14ffa00, {0xc06ae9a000?, 0x1c421a8?, 0x1c43665?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc09e1c1548, {0xc06ae9a000?, 0xc54d73c840?, 0x155?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0d0b87a40, {0xc06a706ac0, 0x9, 0x1c39d7d?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0d0b87a40}, {0xc06a706ac0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc06a706ac0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0d0b87a40?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc06a706a80) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc06a5c9200, 0xc0ae4bafa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323887 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 322231 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c79e0, {0x657b950, 0xc004b29450}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11174008 [select, 3448 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0e887efa0, {0x657b950, 0xc0b41115e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322205 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21eb20, {0x657b950, 0xc003ecd590}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614504 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad0410708, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc349a48300?, 0xc1995da000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc349a48300, {0xc1995da000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc349a48300, {0xc1995da000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc4019382b0, {0xc1995da000?, 0xc34c5f40c0?, 0xc02f5f7cd0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc399fd76e0, {0xc197afaf20, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc399fd76e0}, {0xc197afaf20, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc197afaf20, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc399fd76e0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc197afaee0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc196012240, 0xc02f5f7fb8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 57614409 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 20666220 [select]: github.com/tiancaiamao/gp.workerLoop(0xc0010e0240) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:70 +0xb0 github.com/tiancaiamao/gp.worker(0xc0010e0240, 0xc02f5f2fa0?) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:51 +0x8d created by github.com/tiancaiamao/gp.(*Pool).Go in goroutine 810 /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:32 +0xb8 goroutine 322238 [select, 3598 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc009bf4400, {0x657b950, 0xc00505c140}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148758144 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e46218, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc2a2cfc180?, 0xc004de4000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc2a2cfc180, {0xc004de4000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc2a2cfc180, {0xc004de4000?, 0xc26b3ed0c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc1bd7b8748, {0xc004de4000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5284f6c00, {0xc5945e4420, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc5627bdbd8}, {0xc5945e4420, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc44b5767e0) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc44b5767e0) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc00af7b380) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc00af7b380, {0x657b918, 0xc44a2c5c80}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc00af7b380) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 322230 [select, 3598 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00a1c7980, {0x657b950, 0xc004b29400}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532385 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc139e992a0, {0x657b950, 0xc1b557b3b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532487 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0a33657a0, {0x657b950, 0xc16d56b360}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148779921 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0b9938fd0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 164440305 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc6292d2060, 0xc6a8afd860) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 164440240 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 148760360 [select, 411 minutes]: context.(*cancelCtx).propagateCancel.func2() /usr/local/go/src/context/context.go:505 +0x98 created by context.(*cancelCtx).propagateCancel in goroutine 1129 /usr/local/go/src/context/context.go:504 +0x395 goroutine 164481229 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349cd40, {0x657b950, 0xc4181503c0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481225 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349c700, {0x657b950, 0xc7235c83c0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322306 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00b8ecc80, {0x657b950, 0xc004964a00}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322228 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148758025 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8ad0410518, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc218be3900?, 0xc004efc000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc218be3900, {0xc004efc000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc218be3900, {0xc004efc000?, 0xc0082150c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc2498dbf48, {0xc004efc000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5c89160c0, {0xc2dc7b9210, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc509f651a0}, {0xc2dc7b9210, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc484a80cc0) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc484a80cc0) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc02eaa09c0) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc02eaa09c0, {0x657b918, 0xc44e200cc0}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc02eaa09c0) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 11178224 [sync.Cond.Wait, 1209 minutes]: sync.runtime_notifyListWait(0xc0c277d550, 0x24) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc002129f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc030a69680) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178264 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164439967 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0de373d40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440300 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 12509517 [IO wait, 1225 minutes]: internal/poll.runtime_pollWait(0x7f8aa323d000, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0e70e0300?, 0xc0e7072000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0e70e0300, {0xc0e7072000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0e70e0300, {0xc0e7072000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0e5e1b418, {0xc0e7072000?, 0x8a4c65ea78?, 0xc65edc2f90?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0e6e7ad20, {0xc0e63ffb60, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0e6e7ad20}, {0xc0e63ffb60, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0e63ffb60, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0e6e7ad20?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0e63ffb20) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0e6443440, 0xc015e82fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509554 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 148757995 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e45c48, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc391d81b00?, 0xc0048c8000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc391d81b00, {0xc0048c8000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc391d81b00, {0xc0048c8000?, 0xc26b6ad0c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc24ad6fc68, {0xc0048c8000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5816e99e0, {0xc0e7be8d40, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc693b214d0}, {0xc0e7be8d40, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc483bb5d10) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc483bb5d10) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc2a541d040) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc2a541d040, {0x657b918, 0xc454e311a0}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc2a541d040) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 164440306 [select, 2 minutes]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc668ddd780, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc668ddd780, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 164440240 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 7013962 [select, 3587 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0ea5e9060, {0x657b950, 0xc01f5e5450}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 3168771 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481252 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc6eb6fc050, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc3724ab500) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481230 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 322209 [select, 3717 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21ee80, {0x657b950, 0xc003ecdc20}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11178225 [IO wait, 1209 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e1c330, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0993eda80?, 0xc031302000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0993eda80, {0xc031302000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0993eda80, {0xc031302000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc06f25e538, {0xc031302000?, 0xc360cf17a0?, 0x155?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc066b959e0, {0xc02ff01620, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc066b959e0}, {0xc02ff01620, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc02ff01620, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc066b959e0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc02ff015e0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc030a69680, 0xc030ab67b8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178264 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164440290 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed788a0, {0x657b950, 0xc452164960}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 322354 [select, 3619 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00c21ef60, {0x657b950, 0xc003ecdea0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 322255 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11178387 [sync.Cond.Wait, 1218 minutes]: sync.runtime_notifyListWait(0xc0ce0cde90, 0x23) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc001a54f70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0317b2000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178359 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 57614429 [select, 2349 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 20593059 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 20532482 [select, 2349 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc139e99b20, {0x657b950, 0xc1b557bc20}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440263 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1c140, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc5e7b91600?, 0xc0df65c000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc5e7b91600, {0xc0df65c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc5e7b91600, {0xc0df65c000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc61da25088, {0xc0df65c000?, 0x0?, 0xc6e2e22b40?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc6e4b498c0, {0xc0df5a0f20, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc6e4b498c0}, {0xc0df5a0f20, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0df5a0f20, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc6e4b498c0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0df5a0ee0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0df5a2480, 0xc0d1ef7ab0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440304 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164481167 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc5fea4ca40, {0x657b950, 0xc6360b3ae0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440297 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed78f80, {0x657b950, 0xc61e49a7d0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 12511027 [IO wait, 1216 minutes]: internal/poll.runtime_pollWait(0x7f8ad0446e10, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0f4949100?, 0xc0f4dc8000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0f4949100, {0xc0f4dc8000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0f4949100, {0xc0f4dc8000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0f3ae55a8, {0xc0f4dc8000?, 0x8a5a8bb8b0?, 0xc47c1535f0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0f4d93260, {0xc0f4d9a2e0, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0f4d93260}, {0xc0f4d9a2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0f4d9a2e0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0f4d93260?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f4d9a2a0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0f46d4b40, 0xc00a742fb8?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12510970 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 148780309 [select]: net/http.(*persistConn).writeLoop(0xc3d6ec7320) /usr/local/go/src/net/http/transport.go:2421 +0xe5 created by net/http.(*Transport).dialConn in goroutine 148780291 /usr/local/go/src/net/http/transport.go:1777 +0x16f1 goroutine 164481268 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323c938, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc5c7080300?, 0xc374e5c000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc5c7080300, {0xc374e5c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc5c7080300, {0xc374e5c000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc5e1f4b958, {0xc374e5c000?, 0x8a42fc1fa0?, 0xc72b4dcc00?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc3d2a56960, {0xc374674120, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc3d2a56960}, {0xc374674120, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc374674120, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc3d2a56960?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc3746740e0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc37419d200, 0x5?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481222 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 6404743 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc04f567090, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc01f5d55e0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404682 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 148758146 [IO wait, 411 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e1caf0, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc47dea0700?, 0xc004e88000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc47dea0700, {0xc004e88000, 0x4000, 0x4000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc47dea0700, {0xc004e88000?, 0xc003d530c8?, 0x1d2dfc5?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0de59f448, {0xc004e88000?, 0x72?, 0x0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc5284f6e40, {0xc2dc7b8fbc, 0x4, 0x94f6a80?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 github.com/pingcap/tidb/server.bufferedReadConn.Read(...) /go/src/github.com/pingcap/tidb/server/buffered_read_conn.go:31 io.ReadAtLeast({0x6549fa0, 0xc5e4e7a030}, {0xc2dc7b8fbc, 0x4, 0x4}, 0x4) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 github.com/pingcap/tidb/server.(*packetIO).readOnePacket(0xc44b576960) /go/src/github.com/pingcap/tidb/server/packetio.go:88 +0xb3 github.com/pingcap/tidb/server.(*packetIO).readPacket(0xc44b576960) /go/src/github.com/pingcap/tidb/server/packetio.go:130 +0x53 github.com/pingcap/tidb/server.(*clientConn).readPacket(0xc00af7b6c0) /go/src/github.com/pingcap/tidb/server/conn.go:437 +0x5f github.com/pingcap/tidb/server.(*clientConn).Run(0xc00af7b6c0, {0x657b918, 0xc44b577200}) /go/src/github.com/pingcap/tidb/server/conn.go:1195 +0x1fc github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc00af7b6c0) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 164481269 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc741ef6960, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc37398de30) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164481222 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 20532381 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0c90d3800, {0x657b950, 0xc19171fdb0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164479780 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e1c238, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc015c39280?, 0xc389fd5000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc015c39280, {0xc389fd5000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc015c39280, {0xc389fd5000?, 0x0?, 0x0?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc603c34ba0, {0xc389fd5000?, 0xc000e2fa48?, 0x1d2dfc5?}) /usr/local/go/src/net/net.go:179 +0x45 github.com/soheilhy/cmux.(*bufferedReader).Read(0xc35f107a50, {0xc389fd5000, 0x0?, 0x1000}) /go/cache/github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f github.com/soheilhy/cmux.(*MuxConn).Read(0xc6672d3140?, {0xc389fd5000?, 0xc000e2faf8?, 0x0?}) /go/cache/github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e net/http.(*connReader).Read(0xc6672d3140, {0xc389fd5000, 0x1000, 0x1000}) /usr/local/go/src/net/http/server.go:791 +0x14b bufio.(*Reader).fill(0xc734d17920) /usr/local/go/src/bufio/bufio.go:113 +0x103 bufio.(*Reader).Peek(0xc734d17920, 0x4) /usr/local/go/src/bufio/bufio.go:151 +0x53 net/http.(*conn).serve(0xc74337bc20, {0x657b918, 0xc000b41b90}) /usr/local/go/src/net/http/server.go:2044 +0x75c created by net/http.(*Server).Serve in goroutine 1131 /usr/local/go/src/net/http/server.go:3086 +0x5cb goroutine 11733516 [select, 3468 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0f2b58de0, {0x657b950, 0xc05b0f8b40}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440217 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa2e1c900, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc677553c00?, 0xc0df654000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc677553c00, {0xc0df654000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc677553c00, {0xc0df654000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc49aa231e8, {0xc0df654000?, 0xc6263c7440?, 0xc72b4dc990?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc679c465a0, {0xc0ddb17ee0, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc679c465a0}, {0xc0ddb17ee0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0ddb17ee0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc679c465a0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0ddb17ea0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0dcaeb680, 0xc764d70001?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440292 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 20532491 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc137a1f1e0, {0x657b950, 0xc1b6c14960}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 12509692 [sync.Cond.Wait, 1213 minutes]: sync.runtime_notifyListWait(0xc0eac1ca90, 0x20) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc02a41bf70?) /usr/local/go/src/sync/cond.go:70 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0e8c89d40) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1693 +0x235 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 12509848 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 11733515 [select, 3448 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0f2b58d60, {0x657b950, 0xc05b0f8aa0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 148782024 [select, 411 minutes]: github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).SpawnSubscriptionHandler.func1() /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer.go:375 +0xc5 created by github.com/pingcap/tidb/br/pkg/streamhelper.(*CheckpointAdvancer).SpawnSubscriptionHandler in goroutine 742 /go/src/github.com/pingcap/tidb/br/pkg/streamhelper/advancer.go:373 +0x20f goroutine 164481232 [select, 2 minutes]: github.com/tikv/client-go/v2/internal/client.(*batchConn).fetchAllPendingRequests(0xc69c413a00, 0x80) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:225 +0xa9 github.com/tikv/client-go/v2/internal/client.(*batchConn).batchSendLoop(0xc69c413a00, {0x4, 0xa, 0x3, {0x5b716eb, 0x4}, {0x5b704b8, 0x3}, {0x100, 0x1000, ...}, ...}) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_batch.go:320 +0x125 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 164481194 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:362 +0x12cd goroutine 164481223 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349c4c0, {0x657b950, 0xc6b9972690}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481228 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349cce0, {0x657b950, 0xc418150370}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 11408090 [select, 3476 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0effa23c0, {0x657b950, 0xc06ca6ed20}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 11174007 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 6404764 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323cb28, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc044e75580?, 0xc009250000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc044e75580, {0xc009250000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc044e75580, {0xc009250000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0b0cf51e8, {0xc009250000?, 0x10000?, 0x1c9d552?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0c4b50f60, {0xc0c667e900, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0c4b50f60}, {0xc0c667e900, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0c667e900, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0c4b50f60?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c667e8c0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc036764d80, 0xc061cc6960?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 6404633 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 5324034 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0db086640, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc06a74cf50) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323887 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 5323048 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc060a66000) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5323020 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 129173939 [select]: github.com/pingcap/tidb/store/copr.(*mppIterator).nextImpl(0xc6a9c77f40, {0x657b918, 0xc68cbfbb30}) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:538 +0x16a github.com/pingcap/tidb/store/copr.(*mppIterator).Next(0xc6a9c77f40, {0x657b918?, 0xc68cbfbb30?}) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:561 +0x26 github.com/pingcap/tidb/executor.(*MPPExecWithRetry).Next(0xc6aa81e3c0, {0x657b918, 0xc68cbfbb30}) /go/src/github.com/pingcap/tidb/executor/mpp_exec_with_retry.go:125 +0x6b github.com/pingcap/tidb/distsql.(*selectResult).fetchResp(0xc6a9c23c20, {0x657b918, 0xc68cbfbb30}) /go/src/github.com/pingcap/tidb/distsql/select_result.go:371 +0xcd github.com/pingcap/tidb/distsql.(*selectResult).Next(0xc6a9c23c20, {0x657b918, 0xc68cbfbb30}, 0xc5fc475ea0) /go/src/github.com/pingcap/tidb/distsql/select_result.go:441 +0x7d github.com/pingcap/tidb/executor.(*MPPGather).Next(0xc6aa80a580, {0x657b918?, 0xc68cbfbb30?}, 0xc5fc475ea0) /go/src/github.com/pingcap/tidb/executor/mpp_gather.go:116 +0x99 github.com/pingcap/tidb/executor.Next({0x657b918, 0xc68cbfbb30}, {0x657ea80, 0xc6aa80a580}, 0xc5fc475ea0) /go/src/github.com/pingcap/tidb/executor/executor.go:327 +0x299 github.com/pingcap/tidb/executor.(*ProjectionExec).unParallelExecute(0xc5a95faa20, {0x657b918?, 0xc68cbfbb30?}, 0xc4ca93c050) /go/src/github.com/pingcap/tidb/executor/projection.go:196 +0x90 github.com/pingcap/tidb/executor.(*ProjectionExec).Next(0xc5a95faa20, {0x657b918, 0xc68cbfbb30}, 0x1?) /go/src/github.com/pingcap/tidb/executor/projection.go:183 +0x56 github.com/pingcap/tidb/executor.Next({0x657b918, 0xc68cbfbb30}, {0x657e900, 0xc5a95faa20}, 0xc4ca93c050) /go/src/github.com/pingcap/tidb/executor/executor.go:327 +0x299 github.com/pingcap/tidb/executor.(*TopNExec).loadChunksUntilTotalLimit(0xc6aa80a6e0, {0x657b918, 0xc68cbfbb30}) /go/src/github.com/pingcap/tidb/executor/sort.go:443 +0x28d github.com/pingcap/tidb/executor.(*TopNExec).Next(0xc6aa80a6e0, {0x657b918, 0xc68cbfbb30}, 0xc5fc475f40) /go/src/github.com/pingcap/tidb/executor/sort.go:409 +0xa5 github.com/pingcap/tidb/executor.Next({0x657b918, 0xc68cbfbb30}, {0x657e7c0, 0xc6aa80a6e0}, 0xc5fc475f40) /go/src/github.com/pingcap/tidb/executor/executor.go:327 +0x299 github.com/pingcap/tidb/executor.(*ProjectionExec).unParallelExecute(0xc7006ee240, {0x657b918?, 0xc68cbfbb30?}, 0xc718277900) /go/src/github.com/pingcap/tidb/executor/projection.go:196 +0x90 github.com/pingcap/tidb/executor.(*ProjectionExec).Next(0xc7006ee240, {0x657b918, 0xc68cbfbb30}, 0x1?) /go/src/github.com/pingcap/tidb/executor/projection.go:183 +0x56 github.com/pingcap/tidb/executor.Next({0x657b918, 0xc68cbfbb30}, {0x657e900, 0xc7006ee240}, 0xc718277900) /go/src/github.com/pingcap/tidb/executor/executor.go:327 +0x299 github.com/pingcap/tidb/executor.(*ExecStmt).next(0xc6a9fdfa40, {0x657b918, 0xc68cbfbb30}, {0x657e900, 0xc7006ee240}, 0xc4515423e0?) /go/src/github.com/pingcap/tidb/executor/adapter.go:1202 +0x6e github.com/pingcap/tidb/executor.(*recordSet).Next(0xc5fc475f90, {0x657b918?, 0xc68cbfbb30?}, 0xc718277900) /go/src/github.com/pingcap/tidb/executor/adapter.go:151 +0xb2 github.com/pingcap/tidb/server.(*tidbResultSet).Next(0x5b74bd8?, {0x657b918?, 0xc68cbfbb30?}, 0x5bb70fb?) /go/src/github.com/pingcap/tidb/server/driver_tidb.go:490 +0x25 github.com/pingcap/tidb/server.(*clientConn).writeChunks(0xc450441a00, {0x657b918, 0xc68cbfbb30}, {0x6585e28, 0xc5fcc33340}, 0x1, 0xfb7f?) /go/src/github.com/pingcap/tidb/server/conn.go:2400 +0x190 github.com/pingcap/tidb/server.(*clientConn).writeResultSet(0xc450441a00, {0x657b918, 0xc68cbfbb30}, {0x6585e28, 0xc5fcc33340}, 0x40?, 0x3, 0x40?) /go/src/github.com/pingcap/tidb/server/conn.go:2343 +0x2f0 github.com/pingcap/tidb/server.(*clientConn).executePreparedStmtAndWriteResult(0xc450441a00, {0x657b918, 0xc68cbfbb30}, {0x65a8ba0, 0xc50e918a80?}, {0xc7302254e0, 0x2, 0x2}, 0x0) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:426 +0xb6a github.com/pingcap/tidb/server.(*clientConn).executePlanCacheStmt(0xc450441a00, {0x657b950, 0xc546865310}, {0x59396c0?, 0xc50e918a80}, {0xc7302254e0, 0x2, 0x2}, 0xf2?) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:236 +0x185 github.com/pingcap/tidb/server.(*clientConn).handleStmtExecute(0xc450441a00, {0x657b950, 0xc546865310}, {0xc6aa80c001, 0x56fd, 0x56fd}) /go/src/github.com/pingcap/tidb/server/conn_stmt.go:227 +0x8dd github.com/pingcap/tidb/server.(*clientConn).dispatch(0xc450441a00, {0x657b918?, 0xc609634e70?}, {0xc6aa80c000, 0x56fe, 0x56fe}) /go/src/github.com/pingcap/tidb/server/conn.go:1483 +0x139e github.com/pingcap/tidb/server.(*clientConn).Run(0xc450441a00, {0x657b918, 0xc609634e70}) /go/src/github.com/pingcap/tidb/server/conn.go:1234 +0x294 github.com/pingcap/tidb/server.(*Server).onConn(0xc000a66c80, 0xc450441a00) /go/src/github.com/pingcap/tidb/server/server.go:755 +0x8e5 created by github.com/pingcap/tidb/server.(*Server).startNetworkListener in goroutine 960 /go/src/github.com/pingcap/tidb/server/server.go:530 +0x78e goroutine 5322552 [IO wait]: internal/poll.runtime_pollWait(0x7f8aa323cd18, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc09a489900?, 0xc05d44e000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc09a489900, {0xc05d44e000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc09a489900, {0xc05d44e000?, 0xaab2ab2c5ecb0500?, 0xc03d693d78?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0baab3df0, {0xc05d44e000?, 0x0?, 0xc6d23f0de0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0d533fc20, {0xc05d33bd20, 0x9, 0x0?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0d533fc20}, {0xc05d33bd20, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc05d33bd20, 0x9, 0xc0bb5183a5?}, {0x65496a0?, 0xc0d533fc20?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc05d33bce0) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc05cfc2480, 0xc089341950?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 5322541 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 20532493 [select]: github.com/tikv/client-go/v2/tikvrpc.CheckStreamTimeoutLoop(0xc1d6059da0, 0xc1a253eb40) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1235 +0x173 created by github.com/tikv/client-go/v2/internal/client.(*connArray).Init in goroutine 20532273 /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:360 +0x11dd goroutine 148779041 [chan receive, 411 minutes]: go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch.func1() /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:59 +0xce created by go.etcd.io/etcd/client/v3/namespace.(*watcherPrefix).Watch in goroutine 809 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/namespace/watch.go:54 +0x3ac goroutine 164440262 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0df5a2480) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1659 +0x151 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440304 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:387 +0x1e1b goroutine 164499396 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc6c05065a0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:327 +0x7c google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:342 google.golang.org/grpc.(*csAttempt).recvMsg(0xc786f07380, {0x59cbf40?, 0xc7842ae280}, 0xc653cb7e70?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:1070 +0xba google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x200?) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:927 +0x1f google.golang.org/grpc.(*clientStream).withRetry(0xc6c0506120, 0xc61fe58ac0, 0xc61fe58ab0) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:776 +0x13a google.golang.org/grpc.(*clientStream).RecvMsg(0xc6c0506120, {0x59cbf40?, 0xc7842ae280?}) /go/cache/google.golang.org/grpc@v1.59.0/stream.go:926 +0x113 github.com/pingcap/kvproto/pkg/tikvpb.(*tikvEstablishMPPConnectionClient).Recv(0xc537964a30) /go/cache/github.com/pingcap/kvproto@v0.0.0-20240109063850-932639606bcf/pkg/tikvpb/tikvpb.pb.go:2185 +0x46 github.com/tikv/client-go/v2/tikvrpc.(*MPPStreamResponse).Recv(0xc70844d1d0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/tikvrpc/tikvrpc.go:1209 +0x98 github.com/tikv/client-go/v2/internal/client.(*RPCClient).getMPPStreamResponse(0xc000f268c0?, {0x657b950?, 0xc5fc475db0?}, {0x6604ff0, 0xc5b5933478}, 0x0?, 0x34630b8a000, 0xc00442e4e0) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:822 +0xf9 github.com/tikv/client-go/v2/internal/client.(*RPCClient).sendRequest(0xc000f268c0, {0x657b950, 0xc5fc475db0}, {0xc6a8990d20, 0x4b}, 0xc6aa37d040, 0x657b918?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:703 +0xb4d github.com/tikv/client-go/v2/internal/client.(*RPCClient).SendRequest(0xc000f268c0, {0x657b950, 0xc5fc475db0}, {0xc6a8990d20, 0x4b}, 0x0?, 0xc002171110?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client.go:723 +0xaf github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest({{0x6565730?, 0xc000f268c0?}}, {0x657b950, 0xc5fc475db0}, {0xc6a8990d20, 0x4b}, 0xc0021711e0?, 0x34630b8a000) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_interceptor.go:60 +0x210 github.com/tikv/client-go/v2/internal/client.reqCollapse.SendRequest({{0x6565a60?, 0xc000f1eea0?}}, {0x657b950, 0xc5fc475db0}, {0xc6a8990d20, 0x4b}, 0xc002171258?, 0xc61fe59270?) /go/cache/github.com/tikv/client-go/v2@v2.0.8-0.20240318071007-fb2a7c7eaa3c/internal/client/client_collapse.go:74 +0xc3 github.com/pingcap/tidb/store/copr.(*tikvClient).SendRequest(0xc0021712e8?, {0x657b950?, 0xc5fc475db0?}, {0xc6a8990d20?, 0x1c2f5ac?}, 0x280000c0021712e8?, 0x657b9c0?) /go/src/github.com/pingcap/tidb/store/copr/store.go:72 +0x2a github.com/pingcap/tidb/store/copr.(*mppIterator).establishMPPConns(0xc6a9c77f40, 0xc5b5933058, 0xc6585d8090, 0xc6a9b6fb00) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:443 +0x305 github.com/pingcap/tidb/store/copr.(*mppIterator).handleDispatchReq(0xc6a9c77f40, {0x657b950, 0xc5fc475db0}, 0xc5b5933058?, 0xc6585d8090) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:360 +0x19f7 github.com/pingcap/tidb/store/copr.(*mppIterator).run.func2(0x1?) /go/src/github.com/pingcap/tidb/store/copr/mpp.go:194 +0x5b created by github.com/pingcap/tidb/store/copr.(*mppIterator).run in goroutine 164499410 /go/src/github.com/pingcap/tidb/store/copr/mpp.go:190 +0x4f goroutine 20532486 [select, 2349 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0a3365120, {0x657b950, 0xc16d56b310}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 20532483 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc139e99ce0, {0x657b950, 0xc1b557be00}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 57614451 [select, 2349 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /go/cache/google.golang.org/grpc@v1.59.0/stream.go:391 +0x8c created by google.golang.org/grpc.newClientStreamWithParams in goroutine 20593045 /go/cache/google.golang.org/grpc@v1.59.0/stream.go:390 +0xe9c goroutine 11178388 [IO wait, 1218 minutes]: internal/poll.runtime_pollWait(0x7f8aa2e45960, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0de31dd80?, 0xc0317aa000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0de31dd80, {0xc0317aa000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0de31dd80, {0xc0317aa000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc08de2dda0, {0xc0317aa000?, 0x18a4b0a67a8?, 0xc360a82cf0?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0b240b3e0, {0xc0314a8580, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0b240b3e0}, {0xc0314a8580, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0314a8580, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0b240b3e0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0314a8540) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0317b2000, 0xc03169e500?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178359 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 11178600 [select, 1205 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc051a97810, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc03152bc70) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178588 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 164440264 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc65554d8b0, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0dee5d2d0) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440304 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 20532383 [select, 3017 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc139e983e0, {0x657b950, 0xc1b557a6e0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20532273 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440218 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc46a833e00, 0x1) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:418 +0x113 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0de84df10) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/controlbuf.go:552 +0x86 google.golang.org/grpc/internal/transport.newHTTP2Client.func6() /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:451 +0x85 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440292 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:449 +0x2433 goroutine 148778944 [select, 411 minutes]: go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0016eca90, 0xc0b19956b0, 0xc001eb6780) /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:812 +0x245 created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 692 /go/cache/go.etcd.io/etcd/client/v3@v3.5.10/watch.go:562 +0xbfb goroutine 11178095 [IO wait, 1205 minutes]: internal/poll.runtime_pollWait(0x7f8ad0479898, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc0b788e880?, 0xc030a24000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0b788e880, {0xc030a24000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc0b788e880, {0xc030a24000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0f0425d20, {0xc030a24000?, 0xc6673858c0?, 0xc5d2fc2330?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc0f6bca960, {0xc03070cac0, 0x9, 0x1c39c25?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc0f6bca960}, {0xc03070cac0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc03070cac0, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc0f6bca960?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc03070ca80) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0302f1b00, 0xc03c8f0fa0?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 11178174 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164481219 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc4d09f6ee0, {0x657b950, 0xc739a40460}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 22241664 [select]: github.com/tiancaiamao/gp.workerLoop(0xc0010e0240) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:70 +0xb0 github.com/tiancaiamao/gp.worker(0xc0010e0240, 0xc1c8482fa0?) /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:51 +0x8d created by github.com/tiancaiamao/gp.(*Pool).Go in goroutine 20593043 /go/cache/github.com/tiancaiamao/gp@v0.0.0-20221230034425-4025bc8a4d4a/gp.go:32 +0xb8 goroutine 164481169 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc5fea4d400, {0x657b950, 0xc6360b3e00}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440295 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed78ce0, {0x657b950, 0xc452165cc0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164439968 [IO wait]: internal/poll.runtime_pollWait(0x7f8ad040fd58, 0x72) /usr/local/go/src/runtime/netpoll.go:343 +0x85 internal/poll.(*pollDesc).wait(0xc649221480?, 0xc0df794000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc649221480, {0xc0df794000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a net.(*netFD).Read(0xc649221480, {0xc0df794000?, 0x1c421a8?, 0x4?}) /usr/local/go/src/net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc689142350, {0xc0df794000?, 0xc728771020?, 0xc6cc409290?}) /usr/local/go/src/net/net.go:179 +0x45 bufio.(*Reader).Read(0xc64527f4a0, {0xc0dd9b9460, 0x9, 0x1c39c53?}) /usr/local/go/src/bufio/bufio.go:244 +0x197 io.ReadAtLeast({0x65496a0, 0xc64527f4a0}, {0xc0dd9b9460, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:335 +0x90 io.ReadFull(...) /usr/local/go/src/io/io.go:354 golang.org/x/net/http2.readFrameHeader({0xc0dd9b9460, 0x9, 0x21b31a5?}, {0x65496a0?, 0xc64527f4a0?}) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:237 +0x65 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0dd9b9420) /go/cache/golang.org/x/net@v0.20.0/http2/frame.go:498 +0x85 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0de373d40, 0xc6aab52330?) /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:1585 +0x245 created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 164440300 /go/cache/google.golang.org/grpc@v1.59.0/internal/transport/http2_client.go:397 +0x1e93 goroutine 164481220 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc4d09f7140, {0x657b950, 0xc6b9972140}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440303 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed794a0, {0x657b950, 0xc6a3c009b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164481227 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc6f349cb60, {0x657b950, 0xc7235c8af0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164481194 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129 goroutine 164440291 [select, 2 minutes]: google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc67ed78900, {0x657b950, 0xc4521649b0}) /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:83 +0x112 created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 164440240 /go/cache/google.golang.org/grpc@v1.59.0/internal/grpcsync/callback_serializer.go:55 +0x129