containerd-shim-runc-v2 for containerd task exec request, hangs

[huanlongzhou]

Description

I use 500 concurrency in the container to execute the ctr task exec other container command。 As a result, some commands have not been returned, causes the request to remain suspended。

This is containerd-shim-runc-v2 stack

Goroutine 1
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  github.com/containerd/containerd/runtime/v2/shim.reap (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim_unix.go:78)
  github.com/containerd/containerd/runtime/v2/shim.serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:485)
  github.com/containerd/containerd/runtime/v2/shim.run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:431)
  github.com/containerd/containerd/runtime/v2/shim.RunManager (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:234)
  main.main (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/cmd/containerd-shim-runc-v2/main.go:35)
  runtime.main (/usr/local/go/src/runtime/proc.go:255)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.rt0_go (/usr/local/go/src/runtime/asm_amd64.s:218)

Goroutine 2
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.goparkunlock (/usr/local/go/src/runtime/proc.go:372)
  runtime.forcegchelper (/usr/local/go/src/runtime/proc.go:306)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.init.7 (/usr/local/go/src/runtime/proc.go:294)

Goroutine 3
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.goparkunlock (/usr/local/go/src/runtime/proc.go:372)
  runtime.bgsweep (/usr/local/go/src/runtime/mgcsweep.go:182)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcenable (/usr/local/go/src/runtime/mgc.go:181)

Goroutine 4
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.goparkunlock (/usr/local/go/src/runtime/proc.go:372)
  runtime.bgscavenge (/usr/local/go/src/runtime/mgcscavenge.go:314)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcenable (/usr/local/go/src/runtime/mgc.go:182)

Goroutine 5
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.goparkunlock (/usr/local/go/src/runtime/proc.go:372)
  runtime.runfinq (/usr/local/go/src/runtime/mfinal.go:177)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.createfing (/usr/local/go/src/runtime/mfinal.go:157)

Goroutine 6
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv2 (/usr/local/go/src/runtime/chan.go:444)
  github.com/containerd/containerd/runtime/v2/shim.setRuntime.func1 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:151)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/shim.setRuntime (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:150)

Goroutine 7
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  runtime.ensureSigM.func1 (/usr/local/go/src/runtime/signal_unix.go:890)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.ensureSigM (/usr/local/go/src/runtime/signal_unix.go:873)

Goroutine 8
  runtime.futex (/usr/local/go/src/runtime/sys_linux_amd64.s:520)
  runtime.futexsleep (/usr/local/go/src/runtime/os_linux.go:44)
  runtime.notetsleep_internal (/usr/local/go/src/runtime/lock_futex.go:183)
  runtime.notetsleepg (/usr/local/go/src/runtime/lock_futex.go:237)
  os/signal.signal_recv (/usr/local/go/src/runtime/sigqueue.go:169)
  os/signal.loop (/usr/local/go/src/os/signal/signal_unix.go:24)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: os/signal.Notify.func1.1 (/usr/local/go/src/os/signal/signal.go:151)

Goroutine 9
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv2 (/usr/local/go/src/runtime/chan.go:444)
  github.com/containerd/containerd/runtime/v2/shim.(*RemoteEventsPublisher).processQ
  
Goroutine 12
  syscall.Syscall6 (/usr/local/go/src/syscall/asm_linux_amd64.s:52)
  golang.org/x/sys/unix.EpollWait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go:56)
  github.com/containerd/containerd/pkg/oom/v1.(*epoller).Run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/oom/v1/v1.go:78)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/runc/task.NewTaskService (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:74)

Goroutine 13
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv2 (/usr/local/go/src/runtime/chan.go:444)
  github.com/containerd/containerd/runtime/v2/runc/task.(*service).processExits (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:512)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/runc/task.NewTaskService (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:83)

Goroutine 14
  syscall.Syscall6 (/usr/local/go/src/syscall/asm_linux_amd64.s:52)
  golang.org/x/sys/unix.EpollWait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go:56)
  github.com/containerd/console.(*Epoller).Wait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/console/console_linux.go:111)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/runc.NewPlatform (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/platform.go:54)

Goroutine 15
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv2 (/usr/local/go/src/runtime/chan.go:444)
  github.com/containerd/containerd/runtime/v2/runc/task.(*service).forward (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:607)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/runc/task.NewTaskService (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:88)

Goroutine 16
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.netpollblock (/usr/local/go/src/runtime/netpoll.go:454)
  internal/poll.runtime_pollWait (/usr/local/go/src/runtime/netpoll.go:234)
  internal/poll.(*pollDesc).wait (/usr/local/go/src/internal/poll/fd_poll_runtime.go:84)
  internal/poll.(*pollDesc).waitRead (/usr/local/go/src/internal/poll/fd_poll_runtime.go:89)
  internal/poll.(*FD).Accept (/usr/local/go/src/internal/poll/fd_unix.go:402)
  net.(*netFD).accept (/usr/local/go/src/net/fd_unix.go:173)
  net.(*UnixListener).accept (/usr/local/go/src/net/unixsock_posix.go:167)
  net.(*UnixListener).Accept (/usr/local/go/src/net/unixsock.go:260)
  github.com/containerd/ttrpc.(*Server).Serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:87)
  github.com/containerd/containerd/runtime/v2/shim.serve.func1 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:468)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/shim.serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:466)


Goroutine 17
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv2 (/usr/local/go/src/runtime/chan.go:444)
  github.com/containerd/containerd/runtime/v2/shim.serve.func2 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:479)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/shim.serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:478)

Goroutine 18
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  github.com/containerd/containerd/runtime/v2/shim.handleExitSignals (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim_unix.go:100)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/runtime/v2/shim.serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/shim/shim.go:484)

Goroutine 21
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.gcBgMarkWorker (/usr/local/go/src/runtime/mgc.go:1200)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcBgMarkStartWorkers (/usr/local/go/src/runtime/mgc.go:1124)

Goroutine 33
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  github.com/containerd/ttrpc.(*serverConn).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:431)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*Server).Serve (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:127)

Goroutine 34
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  context.propagateCancel.func1 (/usr/local/go/src/context/context.go:279)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: context.propagateCancel (/usr/local/go/src/context/context.go:278)

Goroutine 35
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.netpollblock (/usr/local/go/src/runtime/netpoll.go:454)
  internal/poll.runtime_pollWait (/usr/local/go/src/runtime/netpoll.go:234)
  internal/poll.(*pollDesc).wait (/usr/local/go/src/internal/poll/fd_poll_runtime.go:84)
  internal/poll.(*pollDesc).waitRead (/usr/local/go/src/internal/poll/fd_poll_runtime.go:89)
  internal/poll.(*FD).Read (/usr/local/go/src/internal/poll/fd_unix.go:167)
  net.(*netFD).Read (/usr/local/go/src/net/fd_posix.go:56)
  net.(*conn).Read (/usr/local/go/src/net/net.go:183)
  bufio.(*Reader).Read (/usr/local/go/src/bufio/bufio.go:227)
  io.ReadAtLeast (/usr/local/go/src/io/io.go:328)
  io.ReadFull (/usr/local/go/src/io/io.go:347)
  github.com/containerd/ttrpc.readMessageHeader (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/channel.go:53)
  github.com/containerd/ttrpc.(*channel).recv (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/channel.go:101)
  github.com/containerd/ttrpc.(*serverConn).run.func1 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:362)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*serverConn).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:332)


Goroutine 38
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  github.com/containerd/ttrpc.(*Client).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:331)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.NewClient (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:94)

Goroutine 39
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.selectgo (/usr/local/go/src/runtime/select.go:327)
  github.com/containerd/ttrpc.(*Client).run.func1 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:265)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*Client).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:262)

Goroutine 40
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.netpollblock (/usr/local/go/src/runtime/netpoll.go:454)
  internal/poll.runtime_pollWait (/usr/local/go/src/runtime/netpoll.go:234)
  internal/poll.(*pollDesc).wait (/usr/local/go/src/internal/poll/fd_poll_runtime.go:84)
  internal/poll.(*pollDesc).waitRead (/usr/local/go/src/internal/poll/fd_poll_runtime.go:89)
  internal/poll.(*FD).Read (/usr/local/go/src/internal/poll/fd_unix.go:167)
  net.(*netFD).Read (/usr/local/go/src/net/fd_posix.go:56)
  net.(*conn).Read (/usr/local/go/src/net/net.go:183)
  bufio.(*Reader).Read (/usr/local/go/src/bufio/bufio.go:227)
  io.ReadAtLeast (/usr/local/go/src/io/io.go:328)
  io.ReadFull (/usr/local/go/src/io/io.go:347)
  github.com/containerd/ttrpc.readMessageHeader (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/channel.go:53)
  github.com/containerd/ttrpc.(*channel).recv (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/channel.go:101)
  github.com/containerd/ttrpc.(*Client).run.func2 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:295)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*Client).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/client.go:287)

Goroutine 52
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.gcBgMarkWorker (/usr/local/go/src/runtime/mgc.go:1200)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcBgMarkStartWorkers (/usr/local/go/src/runtime/mgc.go:1124)

Goroutine 53
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.gcBgMarkWorker (/usr/local/go/src/runtime/mgc.go:1200)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcBgMarkStartWorkers (/usr/local/go/src/runtime/mgc.go:1124)

Goroutine 68
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.gcBgMarkWorker (/usr/local/go/src/runtime/mgc.go:1200)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: runtime.gcBgMarkStartWorkers (/usr/local/go/src/runtime/mgc.go:1124)

Goroutine 96
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.netpollblock (/usr/local/go/src/runtime/netpoll.go:454)
  internal/poll.runtime_pollWait (/usr/local/go/src/runtime/netpoll.go:234)
  internal/poll.(*pollDesc).wait (/usr/local/go/src/internal/poll/fd_poll_runtime.go:84)
  internal/poll.(*pollDesc).waitRead (/usr/local/go/src/internal/poll/fd_poll_runtime.go:89)
  internal/poll.(*FD).Read (/usr/local/go/src/internal/poll/fd_unix.go:167)
  os.(*File).read (/usr/local/go/src/os/file_posix.go:32)
  os.(*File).Read (/usr/local/go/src/os/file.go:119)
  github.com/containerd/fifo.(*fifo).Read (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/fifo/fifo.go:181)
  io.copyBuffer (/usr/local/go/src/io/io.go:423)
  io.Copy (/usr/local/go/src/io/io.go:382)
  os.genericReadFrom (/usr/local/go/src/os/file.go:162)
  os.(*File).ReadFrom (/usr/local/go/src/os/file.go:156)
  io.copyBuffer (/usr/local/go/src/io/io.go:409)
  io.CopyBuffer (/usr/local/go/src/io/io.go:396)
  github.com/containerd/containerd/pkg/process.copyPipes.func3 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/io.go:227)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/pkg/process.copyPipes (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/io.go:222)

Goroutine 72552
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv1 (/usr/local/go/src/runtime/chan.go:439)
  github.com/containerd/containerd/pkg/process.(*execProcess).Wait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/exec.go:65)
  github.com/containerd/containerd/runtime/v2/runc/task.(*service).Wait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:440)
  github.com/containerd/containerd/runtime/v2/task.RegisterTaskService.func14 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/task/shim.pb.go:3547)
  github.com/containerd/ttrpc.defaultServerInterceptor (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/interceptor.go:45)
  github.com/containerd/ttrpc.(*serviceSet).dispatch (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:95)
  github.com/containerd/ttrpc.(*serviceSet).call (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:64)
  github.com/containerd/ttrpc.(*serverConn).run.func2 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:438)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*serverConn).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:434)

Goroutine 76066
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.chanrecv (/usr/local/go/src/runtime/chan.go:576)
  runtime.chanrecv1 (/usr/local/go/src/runtime/chan.go:439)
  github.com/containerd/containerd/pkg/process.(*execProcess).Wait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/exec.go:65)
  github.com/containerd/containerd/runtime/v2/runc/task.(*service).Wait (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/runc/task/service.go:440)
  github.com/containerd/containerd/runtime/v2/task.RegisterTaskService.func14 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/task/shim.pb.go:3547)
  github.com/containerd/ttrpc.defaultServerInterceptor (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/interceptor.go:45)
  github.com/containerd/ttrpc.(*serviceSet).dispatch (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:95)
  github.com/containerd/ttrpc.(*serviceSet).call (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:64)
  github.com/containerd/ttrpc.(*serverConn).run.func2 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:438)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*serverConn).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:434)
  github.com/containerd/containerd/runtime/v2/task.RegisterTaskService.func14 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/runtime/v2/task/shim.pb.go:3547)
  github.com/containerd/ttrpc.defaultServerInterceptor (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/interceptor.go:45)
  github.com/containerd/ttrpc.(*serviceSet).dispatch (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:95)
  github.com/containerd/ttrpc.(*serviceSet).call (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/services.go:64)
  github.com/containerd/ttrpc.(*serverConn).run.func2 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:438)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/ttrpc.(*serverConn).run (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/ttrpc/server.go:434)

Goroutine 81881
  runtime.gopark (/usr/local/go/src/runtime/proc.go:367)
  runtime.netpollblock (/usr/local/go/src/runtime/netpoll.go:454)
  internal/poll.runtime_pollWait (/usr/local/go/src/runtime/netpoll.go:234)
  internal/poll.(*pollDesc).wait (/usr/local/go/src/internal/poll/fd_poll_runtime.go:84)
  internal/poll.(*pollDesc).waitRead (/usr/local/go/src/internal/poll/fd_poll_runtime.go:89)
  internal/poll.(*FD).Read (/usr/local/go/src/internal/poll/fd_unix.go:167)
  os.(*File).read (/usr/local/go/src/os/file_posix.go:32)
  os.(*File).Read (/usr/local/go/src/os/file.go:119)
  github.com/containerd/fifo.(*fifo).Read (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/vendor/github.com/containerd/fifo/fifo.go:181)
  io.copyBuffer (/usr/local/go/src/io/io.go:423)
  io.Copy (/usr/local/go/src/io/io.go:382)
  os.genericReadFrom (/usr/local/go/src/os/file.go:162)
  os.(*File).ReadFrom (/usr/local/go/src/os/file.go:156)
  io.copyBuffer (/usr/local/go/src/io/io.go:409)
  io.CopyBuffer (/usr/local/go/src/io/io.go:396)
  github.com/containerd/containerd/pkg/process.copyPipes.func3 (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/io.go:227)
  runtime.goexit (/usr/local/go/src/runtime/asm_amd64.s:1581)
  created at: github.com/containerd/containerd/pkg/process.copyPipes (/root/rpmbuild/BUILD/container/containerd/containerd-1.6.4/pkg/process/io.go:222)

Steps to reproduce the issue

1. Enter a running container.
2. Use the ctr -n k8s.io task exec --exec-id random number other container /test.sh command on another container in the container. 500 concurrent.
3. test.sh
   `
   #!/bin/bash

sleep 5

exit 0
`

Describe the results you received and expected

containerd-shim-runc-v2 no blocking

What version of containerd are you using?

containerd-1.6.4

Any other relevant information

No response

Show configuration if it is related to CRI plugin.

No response

[fuweid]

@huanlongzhou would you mind to provide more information about how to create the ctr exec?
For the 500 concurrent, does it start 500 ctr exec commands at once? or continue to send the requests?
The ctr exec will fork multiple processes. Please ensure that both the host cpu load and your container cpu quota are enough.
Thanks.

[fuweid]

Closing since there is no update. Please feel free to reopen it.