Improve sub-process handling

- Better backoff when processes exit quickly
- Use PGID to send signals to all subprocesses

cmd/modd/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -58,15 +59,22 @@ func main() {
 
 	if *exec != "" {
 		parser := syntax.NewParser()
-		runner := interp.Runner{
-			Stdout: os.Stdout,
-			Stderr: os.Stderr,
-		}
 		prog, err := parser.Parse(strings.NewReader(*exec), "")
 		if err != nil {
 			os.Exit(1)
 		}
+
+		runner := interp.Runner{
+			Stdin:       os.Stdin,
+			Stdout:      os.Stdout,
+			Stderr:      os.Stderr,
+			KillTimeout: -1,
+		}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		runner.Context = ctx
 		runner.Reset()
+
 		err = runner.Run(prog)
 		if err != nil {
 			os.Exit(1)

daemon.go

@@ -18,18 +18,19 @@ const (
 	// MulRestart is the exponential backoff multiplier applied when the daemon exits uncleanly
 	MulRestart = 2
 	// MaxRestart is the maximum amount of time between daemon restarts
-	MaxRestart = 5 * time.Second
+	MaxRestart = 8 * time.Second
 )
 
 // A single daemon
 type daemon struct {
 	conf  conf.Daemon
 	indir string
 
-	ex    *shell.Executor
-	log   termlog.Stream
-	shell string
-	stop  bool
+	running bool
+	ex      *shell.Executor
+	log     termlog.Stream
+	shell   string
+	stop    bool
 	sync.Mutex
 }
 
@@ -43,20 +44,17 @@ func (d *daemon) Run() {
 	var lastStart time.Time
 	delay := MinRestart
 	for d.stop != true {
-		d.log.Notice(">> starting...")
-		since := time.Now().Sub(lastStart)
-		if since < delay {
-			time.Sleep(delay - since)
+		if !lastStart.IsZero() {
+			d.log.Notice(">> sleeping... %#v", delay)
+			time.Sleep(delay)
 		}
+		d.log.Notice(">> starting...")
 		lastStart = time.Now()
 		err, pstate := ex.Run(d.log, false)
 
-		bump := false
 		if err != nil {
-			bump = true
 			d.log.Shout("execution error: %s", err)
 		} else if pstate.Error != nil {
-			bump = true
 			if _, ok := pstate.Error.(*exec.ExitError); ok {
 				d.log.Warn("exited: %s", pstate.ProcState)
 			} else {
@@ -68,17 +66,23 @@ func (d *daemon) Run() {
 
 		// If we exited cleanly, or the process ran for > MaxRestart, we reset
 		// the delay timer
-		if !bump || (time.Now().Sub(lastStart) > MaxRestart) {
+		if time.Now().Sub(lastStart) > MaxRestart {
 			delay = MinRestart
 		} else {
 			delay *= MulRestart
+			if delay > MaxRestart {
+				delay = MaxRestart
+			}
 		}
 	}
 }
 
 // Restart the daemon, or start it if it's not yet running
 func (d *daemon) Restart() {
-	if d.ex == nil {
+	d.Lock()
+	defer d.Unlock()
+	if !d.running {
+		d.running = true
 		go d.Run()
 	} else {
 		d.log.Notice(">> sending signal %s", d.conf.RestartSignal)
@@ -92,6 +96,7 @@ func (d *daemon) Restart() {
 }
 
 func (d *daemon) Shutdown(sig os.Signal) error {
+	d.log.Notice(">> stopping")
 	d.stop = true
 	if d.ex != nil {
 		return d.ex.Stop()

shell/shell.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"sync"
+	"syscall"
 
 	"github.com/cortesi/termlog"
 	"github.com/google/shlex"
@@ -117,11 +118,15 @@ func (e *Executor) reset() {
 }
 
 func (e *Executor) Run(log termlog.Stream, bufferr bool) (error, *ExecState) {
+	if e.cmd != nil {
+		return fmt.Errorf("already running"), nil
+	}
 	cmd, buff, wg, err := e.start(log, bufferr)
 	if err != nil {
 		return err, nil
 	}
 	eret := cmd.Wait()
+
 	wg.Wait()
 	estate := &ExecState{
 		Error:     eret,
@@ -138,16 +143,11 @@ func (e *Executor) Signal(sig os.Signal) error {
 	if !e.running() {
 		return fmt.Errorf("executor not running")
 	}
-	return e.cmd.Process.Signal(sig)
+	return syscall.Kill(-e.cmd.Process.Pid, sig.(syscall.Signal))
 }
 
 func (e *Executor) Stop() error {
-	e.Lock()
-	defer e.Unlock()
-	if !e.running() {
-		return fmt.Errorf("executor not running")
-	}
-	return e.cmd.Process.Kill()
+	return e.Signal(os.Kill)
 }
 
 func logOutput(wg *sync.WaitGroup, fp io.ReadCloser, out func(string, ...interface{})) {
@@ -190,6 +190,7 @@ func makeCommand(shell string, command string, dir string) (*exec.Cmd, error) {
 		return nil, fmt.Errorf("Unknown shell: %s", shell)
 	}
 	cmd.Dir = dir
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	return cmd, nil
 }