panic handlers everywhere

cmd/server/main-server.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"os/signal"
-	"runtime/debug"
 
 	"runtime"
 	"sync"
@@ -19,6 +18,7 @@ import (
 	"github.com/wavetermdev/waveterm/pkg/authkey"
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/service"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
@@ -111,15 +111,16 @@ func telemetryLoop() {
 	}
 }
 
+func panicTelemetryHandler() {
+	activity := telemetry.ActivityUpdate{NumPanics: 1}
+	err := telemetry.UpdateActivity(context.Background(), activity)
+	if err != nil {
+		log.Printf("error updating activity (panicTelemetryHandler): %v\n", err)
+	}
+}
+
 func sendTelemetryWrapper() {
-	defer func() {
-		r := recover()
-		if r == nil {
-			return
-		}
-		log.Printf("[error] in sendTelemetryWrapper: %v\n", r)
-		debug.PrintStack()
-	}()
+	defer panichandler.PanicHandler("sendTelemetryWrapper")
 	ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancelFn()
 	beforeSendActivityUpdate(ctx)
@@ -254,7 +255,9 @@ func main() {
 		log.Printf("error initializing wstore: %v\n", err)
 		return
 	}
+	panichandler.PanicTelemetryHandler = panicTelemetryHandler
 	go func() {
+		defer panichandler.PanicHandler("InitCustomShellStartupFiles")
 		err := shellutil.InitCustomShellStartupFiles()
 		if err != nil {
 			log.Printf("error initializing wsh and shell-integration files: %v\n", err)

cmd/wsh/cmd/wshcmd-connserver.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/packetparser"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -53,13 +54,15 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
 	var routeIdContainer atomic.Pointer[string]
 	proxy := wshutil.MakeRpcProxy()
 	go func() {
+		defer panichandler.PanicHandler("handleNewListenerConn:AdaptOutputChToStream")
 		writeErr := wshutil.AdaptOutputChToStream(proxy.ToRemoteCh, conn)
 		if writeErr != nil {
 			log.Printf("error writing to domain socket: %v\n", writeErr)
 		}
 	}()
 	go func() {
 		// when input is closed, close the connection
+		defer panichandler.PanicHandler("handleNewListenerConn:AdaptStreamToMsgCh")
 		defer func() {
 			conn.Close()
 			routeIdPtr := routeIdContainer.Load()
@@ -136,6 +139,7 @@ func serverRunRouter() error {
 	rawCh := make(chan []byte, wshutil.DefaultOutputChSize)
 	go packetparser.Parse(os.Stdin, termProxy.FromRemoteCh, rawCh)
 	go func() {
+		defer panichandler.PanicHandler("serverRunRouter:WritePackets")
 		for msg := range termProxy.ToRemoteCh {
 			packetparser.WritePacket(os.Stdout, msg)
 		}

pkg/blockcontroller/blockcontroller.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/shellexec"
@@ -354,6 +355,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 	ptyBuffer := wshutil.MakePtyBuffer(wshutil.WaveOSCPrefix, bc.ShellProc.Cmd, wshProxy.FromRemoteCh)
 	go func() {
 		// handles regular output from the pty (goes to the blockfile and xterm)
+		defer panichandler.PanicHandler("blockcontroller:shellproc-pty-read-loop")
 		defer func() {
 			log.Printf("[shellproc] pty-read loop done\n")
 			bc.ShellProc.Close()
@@ -386,6 +388,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 	go func() {
 		// handles input from the shellInputCh, sent to pty
 		// use shellInputCh instead of bc.ShellInputCh (because we want to be attached to *this* ch.  bc.ShellInputCh can be updated)
+		defer panichandler.PanicHandler("blockcontroller:shellproc-input-loop")
 		for ic := range shellInputCh {
 			if len(ic.InputData) > 0 {
 				bc.ShellProc.Cmd.Write(ic.InputData)
@@ -403,6 +406,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("blockcontroller:shellproc-output-loop")
 		// handles outputCh -> shellInputCh
 		for msg := range wshProxy.ToRemoteCh {
 			encodedMsg, err := wshutil.EncodeWaveOSCBytes(wshutil.WaveServerOSC, msg)
@@ -413,6 +417,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("blockcontroller:shellproc-wait-loop")
 		// wait for the shell to finish
 		defer func() {
 			wshutil.DefaultRouter.UnregisterRoute(wshutil.MakeControllerRouteId(bc.BlockId))
@@ -484,6 +489,7 @@ func (bc *BlockController) run(bdata *waveobj.Block, blockMeta map[string]any, r
 	runOnStart := getBoolFromMeta(blockMeta, waveobj.MetaKey_CmdRunOnStart, true)
 	if runOnStart {
 		go func() {
+			defer panichandler.PanicHandler("blockcontroller:run-shell-command")
 			var termSize waveobj.TermSize
 			if rtOpts != nil {
 				termSize = rtOpts.TermSize

pkg/filestore/blockstore.go

@@ -12,12 +12,12 @@ import (
 	"fmt"
 	"io/fs"
 	"log"
-	"runtime/debug"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/wavetermdev/waveterm/pkg/ijson"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 )
 
 const (
@@ -514,12 +514,7 @@ func (s *FileStore) runFlushWithNewContext() (FlushStats, error) {
 }
 
 func (s *FileStore) runFlusher() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Printf("panic in filestore flusher: %v\n", r)
-			debug.PrintStack()
-		}
-	}()
+	defer panichandler.PanicHandler("filestore flusher")
 	for {
 		stats, err := s.runFlushWithNewContext()
 		if err != nil || stats.NumDirtyEntries > 0 {

pkg/panichandler/panichandler.go

@@ -0,0 +1,43 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package panichandler
+
+import (
+	"fmt"
+	"log"
+	"runtime/debug"
+)
+
+// to log NumPanics into the local telemetry system
+// gets around import cycles
+var PanicTelemetryHandler func()
+
+func PanicHandlerNoTelemetry(debugStr string) {
+	r := recover()
+	if r == nil {
+		return
+	}
+	log.Printf("[panic] in %s: %v\n", debugStr, r)
+	debug.PrintStack()
+}
+
+// returns an error (wrapping the panic) if a panic occurred
+func PanicHandler(debugStr string) error {
+	r := recover()
+	if r == nil {
+		return nil
+	}
+	log.Printf("[panic] in %s: %v\n", debugStr, r)
+	debug.PrintStack()
+	if PanicTelemetryHandler != nil {
+		go func() {
+			defer PanicHandlerNoTelemetry("PanicTelemetryHandler")
+			PanicTelemetryHandler()
+		}()
+	}
+	if err, ok := r.(error); ok {
+		return fmt.Errorf("panic in %s: %w", debugStr, err)
+	}
+	return fmt.Errorf("panic in %s: %v", debugStr, r)
+}

pkg/remote/conncontroller/conncontroller.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/kevinburke/ssh_config"
 	"github.com/skeema/knownhosts"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/pkg/userinput"
@@ -193,6 +194,7 @@ func (conn *SSHConn) OpenDomainSocketListener() error {
 		conn.DomainSockListener = listener
 	})
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:OpenDomainSocketListener")
 		defer conn.WithLock(func() {
 			conn.DomainSockListener = nil
 			conn.SockName = ""
@@ -252,6 +254,7 @@ func (conn *SSHConn) StartConnServer() error {
 	})
 	// service the I/O
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:sshSession.Wait")
 		// wait for termination, clear the controller
 		defer conn.WithLock(func() {
 			conn.ConnController = nil
@@ -260,6 +263,7 @@ func (conn *SSHConn) StartConnServer() error {
 		log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr)
 	}()
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:sshSession-output")
 		readErr := wshutil.StreamToLines(pipeRead, func(line []byte) {
 			lineStr := string(line)
 			if !strings.HasSuffix(lineStr, "\n") {

pkg/remote/connutil.go

@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -288,6 +289,7 @@ func CpHostToRemote(client *ssh.Client, sourcePath string, destPath string) erro
 	}
 
 	go func() {
+		defer panichandler.PanicHandler("connutil:CpHostToRemote")
 		io.Copy(installStdin, input)
 		session.Close() // this allows the command to complete for reasons i don't fully understand
 	}()

pkg/service/objectservice/objectservice.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
 	"github.com/wavetermdev/waveterm/pkg/wcore"
@@ -94,6 +95,7 @@ func (svc *ObjectService) AddTabToWorkspace(windowId string, tabName string, act
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("ObjectService:AddTabToWorkspace:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	return tabId, updates, nil
@@ -142,6 +144,7 @@ func (svc *ObjectService) SetActiveTab(windowId string, tabId string) (waveobj.U
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("ObjectService:SetActiveTab:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	var extraUpdates waveobj.UpdatesRtnType

pkg/service/windowservice/windowservice.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
 	"github.com/wavetermdev/waveterm/pkg/eventbus"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
@@ -75,6 +76,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
 		}
 	}
 	go func() {
+		defer panichandler.PanicHandler("WindowService:CloseTab:StopBlockControllers")
 		for _, blockId := range tab.BlockIds {
 			blockcontroller.StopBlockController(blockId)
 		}
@@ -107,6 +109,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("WindowService:CloseTab:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	return rtn, updates, nil

pkg/shellexec/conninterface.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/creack/pty"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wsl"
 	"golang.org/x/crypto/ssh"
 )
@@ -51,6 +52,7 @@ func (cw CmdWrap) KillGraceful(timeout time.Duration) {
 		cw.Cmd.Process.Signal(syscall.SIGTERM)
 	}
 	go func() {
+		defer panichandler.PanicHandler("KillGraceful:Kill")
 		time.Sleep(timeout)
 		if cw.Cmd.ProcessState == nil || !cw.Cmd.ProcessState.Exited() {
 			cw.Cmd.Process.Kill() // force kill if it is already not exited
@@ -159,6 +161,7 @@ func (wcw WslCmdWrap) KillGraceful(timeout time.Duration) {
 	}
 	process.Signal(os.Interrupt)
 	go func() {
+		defer panichandler.PanicHandler("KillGraceful-wsl:Kill")
 		time.Sleep(timeout)
 		process := wcw.WslCmd.GetProcess()
 		processState := wcw.WslCmd.GetProcessState()

pkg/shellexec/shellexec.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/creack/pty"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/util/shellutil"
@@ -51,6 +52,7 @@ type ShellProc struct {
 func (sp *ShellProc) Close() {
 	sp.Cmd.KillGraceful(DefaultGracefulKillWait)
 	go func() {
+		defer panichandler.PanicHandler("ShellProc.Close")
 		waitErr := sp.Cmd.Wait()
 		sp.SetWaitErrorAndSignalDone(waitErr)
 
@@ -450,6 +452,7 @@ func RunSimpleCmdInPty(ecmd *exec.Cmd, termSize waveobj.TermSize) ([]byte, error
 	ioDone := make(chan bool)
 	var outputBuf bytes.Buffer
 	go func() {
+		panichandler.PanicHandler("RunSimpleCmdInPty:ioCopy")
 		// ignore error (/dev/ptmx has read error when process is done)
 		defer close(ioDone)
 		io.Copy(&outputBuf, cmdPty)

pkg/telemetry/telemetry.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"time"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/daystr"
 	"github.com/wavetermdev/waveterm/pkg/util/dbutil"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
@@ -36,6 +37,7 @@ type ActivityUpdate struct {
 	NumSSHConn    int                   `json:"numsshconn,omitempty"`
 	NumWSLConn    int                   `json:"numwslconn,omitempty"`
 	NumMagnify    int                   `json:"nummagnify,omitempty"`
+	NumPanics     int                   `json:"numpanics,omitempty"`
 	Startup       int                   `json:"startup,omitempty"`
 	Shutdown      int                   `json:"shutdown,omitempty"`
 	SetTabTheme   int                   `json:"settabtheme,omitempty"`
@@ -71,6 +73,7 @@ type TelemetryData struct {
 	NewTab        int                   `json:"newtab"`
 	NumStartup    int                   `json:"numstartup,omitempty"`
 	NumShutdown   int                   `json:"numshutdown,omitempty"`
+	NumPanics     int                   `json:"numpanics,omitempty"`
 	SetTabTheme   int                   `json:"settabtheme,omitempty"`
 	Displays      []ActivityDisplayType `json:"displays,omitempty"`
 	Renderers     map[string]int        `json:"renderers,omitempty"`
@@ -104,6 +107,7 @@ func AutoUpdateChannel() string {
 // Wraps UpdateCurrentActivity, spawns goroutine, and logs errors
 func GoUpdateActivityWrap(update ActivityUpdate, debugStr string) {
 	go func() {
+		defer panichandler.PanicHandlerNoTelemetry("GoUpdateActivityWrap")
 		ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancelFn()
 		err := UpdateActivity(ctx, update)
@@ -138,6 +142,7 @@ func UpdateActivity(ctx context.Context, update ActivityUpdate) error {
 		tdata.NumShutdown += update.Shutdown
 		tdata.SetTabTheme += update.SetTabTheme
 		tdata.NumMagnify += update.NumMagnify
+		tdata.NumPanics += update.NumPanics
 		if update.NumTabs > 0 {
 			tdata.NumTabs = update.NumTabs
 		}