Skip to content

Commit

Permalink
feat(sidecar): stop waiting for workers to finish
Browse files Browse the repository at this point in the history
1. Log an error every 5 seconds when waiting.
2. Panic if we wait too long.

We don't want to just move on because we have a goroutine hanging.
  • Loading branch information
Stebalien committed Feb 4, 2020
1 parent cc1d39f commit c1060ca
Showing 1 changed file with 21 additions and 1 deletion.
22 changes: 21 additions & 1 deletion pkg/dockermanager/dockermanager.go
Expand Up @@ -13,6 +13,9 @@ import (
"github.com/ipfs/testground/pkg/logging"
)

const workerShutdownTimeout = 1 * time.Minute
const workerShutdownTick = 5 * time.Second

// Manager is a convenient wrapper around the docker client.
type Manager struct {
logging.Logging
Expand Down Expand Up @@ -123,7 +126,24 @@ func (dm *Manager) Manage(
if m, ok := managers[container]; ok {
m.cancel()
delete(managers, container)
<-m.done

timeout := time.NewTimer(workerShutdownTimeout)
defer timeout.Stop()

ticker := time.NewTicker(workerShutdownTick)
defer ticker.Stop()

for {
select {
case <-m.done:
return
case <-timeout.C:
dm.S().Panicw("timed out waiting for container worker to stop", "container", container)
return
case <-ticker.C:
dm.S().Errorw("waiting for container worker to stop", "container", container)
}
}
}
}
start := func(container string) {
Expand Down

0 comments on commit c1060ca

Please sign in to comment.