Skip to content

Commit

Permalink
fix shim reaper wait command execute blocked
Browse files Browse the repository at this point in the history
wait no timeout will lead to event publish
process hang in some special scenarios.

Signed-off-by: botieking98 <botieking@gmail.com>
  • Loading branch information
botieking committed Oct 27, 2021
1 parent aa65fae commit 3e51312
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 1 deletion.
2 changes: 1 addition & 1 deletion cmd/containerd-shim/main_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event
if err != nil {
return err
}
status, err := reaper.Default.Wait(cmd, c)
status, err := reaper.Default.WaitTimeout(cmd, c, 30*time.Second)
if err != nil {
return errors.Wrapf(err, "failed to publish event: %s", b.String())
}
Expand Down
23 changes: 23 additions & 0 deletions sys/reaper/reaper_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package reaper

import (
"sync"
"syscall"
"time"

runc "github.com/containerd/go-runc"
Expand Down Expand Up @@ -116,6 +117,28 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
return -1, ErrNoSuchProcess
}

// WaitTimeout is used to skip the blocked command and kill the left process.
func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, timeout time.Duration) (int, error) {
sch := make(chan int)
ech := make(chan error)
go func() {
status, err := m.Wait(c, ec)
sch <- status
if err != nil {
ech <- err
}
}()
select {
case <-time.After(timeout):
syscall.Kill(c.Process.Pid, syscall.SIGKILL)
return 0, errors.Errorf("timeout %ds for cmd(pid=%d): %s, %s", timeout/time.Second, c.Process.Pid, c.Path, c.Args)
case status := <-sch:
return status, nil
case err := <-ech:
return -1, err
}
}

// Subscribe to process exit changes
func (m *Monitor) Subscribe() chan runc.Exit {
c := make(chan runc.Exit, bufferSize)
Expand Down

0 comments on commit 3e51312

Please sign in to comment.