feat: failsafe mode

cmd/main.go

@@ -16,10 +16,10 @@ import (
 )
 
 const (
-	envChildID         = "JETKVM_CHILD_ID"
-	errorDumpDir       = "/userdata/jetkvm/"
-	errorDumpStateFile = ".has_error_dump"
-	errorDumpTemplate  = "jetkvm-%s.log"
+	envChildID        = "JETKVM_CHILD_ID"
+	errorDumpDir      = "/userdata/jetkvm/crashdump"
+	errorDumpLastFile = "last-crash.log"
+	errorDumpTemplate = "jetkvm-%s.log"
 )
 
 func program() {
@@ -74,7 +74,12 @@ func supervise() error {
 	// run the child binary
 	cmd := exec.Command(binPath)
 
-	cmd.Env = append(os.Environ(), []string{envChildID + "=" + kvm.GetBuiltAppVersion()}...)
+	lastFilePath := filepath.Join(errorDumpDir, errorDumpLastFile)
+
+	cmd.Env = append(os.Environ(), []string{
+		fmt.Sprintf("%s=%s", envChildID, kvm.GetBuiltAppVersion()),
+		fmt.Sprintf("JETKVM_LAST_ERROR_PATH=%s", lastFilePath),
+	}...)
 	cmd.Args = os.Args
 
 	logFile, err := os.CreateTemp("", "jetkvm-stdout.log")
@@ -117,49 +122,108 @@ func supervise() error {
 	return nil
 }
 
-func createErrorDump(logFile *os.File) {
-	logFile.Close()
+func isSymlinkTo(oldName, newName string) bool {
+	file, err := os.Stat(newName)
+	if err != nil {
+		return false
+	}
+	if file.Mode()&os.ModeSymlink != os.ModeSymlink {
+		return false
+	}
+	target, err := os.Readlink(newName)
+	if err != nil {
+		return false
+	}
+	return target == oldName
+}
 
-	// touch the error dump state file
-	if err := os.WriteFile(filepath.Join(errorDumpDir, errorDumpStateFile), []byte{}, 0644); err != nil {
-		return
+func ensureSymlink(oldName, newName string) error {
+	if isSymlinkTo(oldName, newName) {
+		return nil
 	}
+	_ = os.Remove(newName)
+	return os.Symlink(oldName, newName)
+}
 
-	fileName := fmt.Sprintf(errorDumpTemplate, time.Now().Format("20060102150405"))
-	filePath := filepath.Join(errorDumpDir, fileName)
-	if err := os.Rename(logFile.Name(), filePath); err == nil {
-		fmt.Printf("error dump created: %s\n", filePath)
-		return
+func renameFile(f *os.File, newName string) error {
+	_ = f.Close()
+
+	// try to rename the file first
+	if err := os.Rename(f.Name(), newName); err == nil {
+		return nil
 	}
 
-	fnSrc, err := os.Open(logFile.Name())
+	// copy the log file to the error dump directory
+	fnSrc, err := os.Open(f.Name())
 	if err != nil {
-		return
+		return fmt.Errorf("failed to open file: %w", err)
 	}
 	defer fnSrc.Close()
 
-	fnDst, err := os.Create(filePath)
+	fnDst, err := os.Create(newName)
 	if err != nil {
-		return
+		return fmt.Errorf("failed to create file: %w", err)
 	}
 	defer fnDst.Close()
 
 	buf := make([]byte, 1024*1024)
 	for {
 		n, err := fnSrc.Read(buf)
 		if err != nil && err != io.EOF {
-			return
+			return fmt.Errorf("failed to read file: %w", err)
 		}
 		if n == 0 {
 			break
 		}
 
 		if _, err := fnDst.Write(buf[:n]); err != nil {
-			return
+			return fmt.Errorf("failed to write file: %w", err)
 		}
 	}
 
-	fmt.Printf("error dump created: %s\n", filePath)
+	return nil
+}
+
+func ensureErrorDumpDir() error {
+	// TODO: check if the directory is writable
+	f, err := os.Stat(errorDumpDir)
+	if err == nil && f.IsDir() {
+		return nil
+	}
+	if err := os.MkdirAll(errorDumpDir, 0755); err != nil {
+		return fmt.Errorf("failed to create error dump directory: %w", err)
+	}
+	return nil
+}
+
+func createErrorDump(logFile *os.File) {
+	fmt.Println()
+
+	fileName := fmt.Sprintf(
+		errorDumpTemplate,
+		time.Now().Format("20060102-150405"),
+	)
+
+	// check if the directory exists
+	if err := ensureErrorDumpDir(); err != nil {
+		fmt.Printf("failed to ensure error dump directory: %v\n", err)
+		return
+	}
+
+	filePath := filepath.Join(errorDumpDir, fileName)
+	if err := renameFile(logFile, filePath); err != nil {
+		fmt.Printf("failed to rename file: %v\n", err)
+		return
+	}
+
+	fmt.Printf("error dump copied: %s\n", filePath)
+
+	lastFilePath := filepath.Join(errorDumpDir, errorDumpLastFile)
+
+	if err := ensureSymlink(filePath, lastFilePath); err != nil {
+		fmt.Printf("failed to create symlink: %v\n", err)
+		return
+	}
 }
 
 func doSupervise() {

failsafe.go

@@ -0,0 +1,107 @@
+package kvm
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+)
+
+const (
+	failsafeDefaultLastCrashPath = "/userdata/jetkvm/crashdump/last-crash.log"
+	failsafeFile                 = "/userdata/jetkvm/.enablefailsafe"
+	failsafeLastCrashEnv         = "JETKVM_LAST_ERROR_PATH"
+	failsafeEnv                  = "JETKVM_FORCE_FAILSAFE"
+)
+
+var (
+	failsafeOnce       sync.Once
+	failsafeCrashLog   = ""
+	failsafeModeActive = false
+	failsafeModeReason = ""
+)
+
+type FailsafeModeNotification struct {
+	Active bool   `json:"active"`
+	Reason string `json:"reason"`
+}
+
+// this function has side effects and can be only executed once
+func checkFailsafeReason() {
+	failsafeOnce.Do(func() {
+		// check if the failsafe environment variable is set
+		if os.Getenv(failsafeEnv) == "1" {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_env_set"
+			return
+		}
+
+		// check if the failsafe file exists
+		if _, err := os.Stat(failsafeFile); err == nil {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_file_exists"
+			_ = os.Remove(failsafeFile)
+			return
+		}
+
+		// get the last crash log path from the environment variable
+		lastCrashPath := os.Getenv(failsafeLastCrashEnv)
+		if lastCrashPath == "" {
+			lastCrashPath = failsafeDefaultLastCrashPath
+		}
+
+		// check if the last crash log file exists
+		l := failsafeLogger.With().Str("path", lastCrashPath).Logger()
+		fi, err := os.Lstat(lastCrashPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				l.Warn().Err(err).Msg("failed to stat last crash log")
+			}
+			return
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			l.Warn().Msg("last crash log is not a symlink, ignoring")
+			return
+		}
+
+		// open the last crash log file and find if it contains the string "panic"
+		content, err := os.ReadFile(lastCrashPath)
+		if err != nil {
+			l.Warn().Err(err).Msg("failed to read last crash log")
+			return
+		}
+
+		// unlink the last crash log file
+		failsafeCrashLog = string(content)
+		_ = os.Remove(lastCrashPath)
+
+		// TODO: read the goroutine stack trace and check which goroutine is panicking
+		if strings.Contains(failsafeCrashLog, "runtime.cgocall") {
+			failsafeModeActive = true
+			failsafeModeReason = "video"
+			return
+		}
+	})
+}
+
+func notifyFailsafeMode(session *Session) {
+	if !failsafeModeActive || session == nil {
+		return
+	}
+
+	jsonRpcLogger.Info().Str("reason", failsafeModeReason).Msg("sending failsafe mode notification")
+
+	writeJSONRPCEvent("failsafeMode", FailsafeModeNotification{
+		Active: true,
+		Reason: failsafeModeReason,
+	}, session)
+}
+
+func rpcGetFailsafeLogs() (string, error) {
+	if !failsafeModeActive {
+		return "", fmt.Errorf("failsafe mode is not active")
+	}
+
+	return failsafeCrashLog, nil
+}