Only reap zombie stunnel processes

go.mod

@@ -4,9 +4,8 @@ require (
 	github.com/aws/aws-sdk-go v1.31.2
 	github.com/container-storage-interface/spec v1.2.0
 	github.com/golang/mock v1.4.4
-	github.com/google/uuid v1.1.1
 	github.com/kubernetes-csi/csi-test v1.1.1
-	github.com/kubernetes-csi/csi-test/v3 v3.1.1 // indirect
+	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/onsi/ginkgo v1.12.2
 	github.com/onsi/gomega v1.10.1
 	google.golang.org/grpc v1.26.0

hack/values.yaml

@@ -1,5 +1,8 @@
 controller:
   create: true
+  logLevel: 5
+node:
+  logLevel: 5
 serviceAccount:
   controller:
     create: true

pkg/driver/driver.go

@@ -115,13 +115,13 @@ func (d *Driver) Run() error {
 	klog.Info("Registering Controller Server")
 	csi.RegisterControllerServer(d.srv, d)
 
-	klog.Info("Starting watchdog")
+	klog.Info("Starting efs-utils watchdog")
 	if err := d.efsWatchdog.start(); err != nil {
 		return err
 	}
 
 	reaper := newReaper()
-	klog.Info("Staring subreaper")
+	klog.Info("Starting reaper")
 	reaper.start()
 
 	klog.Infof("Listening for connections on address: %#v", listener.Addr())

pkg/driver/reaper.go

@@ -17,10 +17,14 @@ limitations under the License.
 package driver
 
 import (
+	"fmt"
+	"io/ioutil"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
+	"github.com/mitchellh/go-ps"
 	"k8s.io/klog"
 )
 
@@ -45,20 +49,29 @@ func (r *reaper) start() {
 	go r.runLoop()
 }
 
-// runLoop waits for all child processes that exit
-// currently only stunnel process is created by efs mount helper
-// and is inherited as the child process of the driver
+// runLoop catches SIGCHLD signals to remove stunnel zombie processes. stunnel
+// processes are guaranteed to become zombies because their parent is the
+// driver process but they get terminated by the efs-utils watchdog process:
+// https://github.com/aws/efs-utils/blob/v1.31.2/src/watchdog/__init__.py#L608
+// pid_max is 4194303.
 func (r *reaper) runLoop() {
 	for {
 		select {
 		case <-r.sigs:
-			var status syscall.WaitStatus
-			var rusage syscall.Rusage
-			childPid, err := syscall.Wait4(-1, &status, syscall.WNOHANG, &rusage)
+			procs, err := ps.Processes()
 			if err != nil {
-				klog.Warningf("Failed to wait for child process %s", err)
+				klog.Warningf("reaper: failed to get all procs: %v", err)
 			} else {
-				klog.V(4).Infof("Waited for child process %d", childPid)
+				for _, p := range procs {
+					reaped := waitIfZombieStunnel(p)
+					if reaped {
+						// wait for only one process per SIGCHLD received over channel. It
+						// doesn't have to be the same process that triggered the
+						// particular SIGCHLD (there's no way to tell anyway), the
+						// intention is to reap zombies as they come.
+						break
+					}
+				}
 			}
 		case <-r.stopCh:
 			break
@@ -70,3 +83,34 @@ func (r *reaper) runLoop() {
 func (r *reaper) stop() {
 	r.stopCh <- struct{}{}
 }
+
+func waitIfZombieStunnel(p ps.Process) bool {
+	if !strings.Contains(p.Executable(), "stunnel") {
+		return false
+	}
+	data, err := ioutil.ReadFile(fmt.Sprintf("/proc/%v/status", p.Pid()))
+	if err != nil {
+		klog.Warningf("reaper: failed to read process %v's status: %v", p, err)
+		return false
+	}
+
+	lines := strings.Split(string(data), "\n")
+	for _, line := range lines {
+		if strings.HasPrefix(line, "State") {
+			if strings.Contains(line, "zombie") {
+				var wstatus syscall.WaitStatus
+				var rusage syscall.Rusage
+				_, err := syscall.Wait4(p.Pid(), &wstatus, 0, &rusage)
+				if err != nil {
+					klog.Warningf("reaper: failed to wait for process %v: %v", p, err)
+				}
+				klog.V(4).Infof("reaper: waited for process %v", p)
+				return true
+			} else {
+				return false
+			}
+		}
+	}
+
+	return false
+}