Implement Lambda runtime init error reporting

cmd/localstack/custom_interop.go

@@ -93,19 +93,18 @@ type InvokeRequest struct {
 type ErrorResponse struct {
 	ErrorMessage string   `json:"errorMessage"`
 	ErrorType    string   `json:"errorType,omitempty"`
-	RequestId    string   `json:"requestId,omitempty"`
+	// RequestId uses *string so that an empty string "" is serialized (not omitted),
+	// while nil is omitted — init errors always set this field, fault events leave it nil.
+	RequestId *string  `json:"requestId,omitempty"`
 	StackTrace   []string `json:"stackTrace,omitempty"`
 }
 
-func NewCustomInteropServer(lsOpts *LsOpts, delegate interop.Server, logCollector *LogCollector) (server *CustomInteropServer) {
+func NewCustomInteropServer(lsOpts *LsOpts, adapter *LocalStackAdapter, delegate interop.Server, logCollector *LogCollector) (server *CustomInteropServer) {
 	server = &CustomInteropServer{
-		delegate:         delegate.(*rapidcore.Server),
-		port:             lsOpts.InteropPort,
-		upstreamEndpoint: lsOpts.RuntimeEndpoint,
-		localStackAdapter: &LocalStackAdapter{
-			UpstreamEndpoint: lsOpts.RuntimeEndpoint,
-			RuntimeId:        lsOpts.RuntimeId,
-		},
+		delegate:          delegate.(*rapidcore.Server),
+		port:              lsOpts.InteropPort,
+		upstreamEndpoint:  lsOpts.RuntimeEndpoint,
+		localStackAdapter: adapter,
 	}
 
 	// TODO: extract this
@@ -219,12 +218,45 @@ func (c *CustomInteropServer) SendErrorResponse(invokeID string, resp *interop.E
 	return c.delegate.SendErrorResponse(invokeID, resp)
 }
 
-// SendInitErrorResponse writes error response during init to a shared memory and sends GIRD FAULT.
+// SendInitErrorResponse forwards the init error to LocalStack and then propagates it to the delegate.
 func (c *CustomInteropServer) SendInitErrorResponse(resp *interop.ErrorInvokeResponse) error {
 	log.Traceln("SendInitErrorResponse called")
-	if err := c.localStackAdapter.SendStatus(Error, resp.Payload); err != nil {
-		log.Fatalln("Failed to send init error to LocalStack " + err.Error() + ". Exiting.")
+
+	// Deserialize the raw payload so we can include the requestId and structured fields.
+	var parsed struct {
+		ErrorMessage string   `json:"errorMessage"`
+		ErrorType    string   `json:"errorType"`
+		StackTrace   []string `json:"stackTrace,omitempty"`
+	}
+	if err := json.Unmarshal(resp.Payload, &parsed); err != nil {
+		log.WithError(err).Warn("Failed to parse init error payload; forwarding raw payload")
+		if err := c.localStackAdapter.SendStatus(Error, resp.Payload); err != nil {
+			log.WithError(err).WithField("runtime-id", c.localStackAdapter.RuntimeId).
+				Error("Failed to send init error to LocalStack")
+		}
+		return c.delegate.SendInitErrorResponse(resp)
+	}
+
+	requestId := c.delegate.GetCurrentInvokeID()
+	adaptedResp := ErrorResponse{
+		ErrorMessage: parsed.ErrorMessage,
+		ErrorType:    parsed.ErrorType,
+		RequestId:    &requestId,
+		StackTrace:   parsed.StackTrace,
+	}
+	body, err := json.Marshal(adaptedResp)
+	if err != nil {
+		log.WithError(err).Error("Failed to marshal adapted init error response")
+		body = resp.Payload
 	}
+
+	go func() {
+		if err := c.localStackAdapter.SendStatus(Error, body); err != nil {
+			log.WithError(err).WithField("runtime-id", c.localStackAdapter.RuntimeId).
+				Error("Failed to send init error to LocalStack")
+		}
+	}()
+
 	return c.delegate.SendInitErrorResponse(resp)
 }
 

cmd/localstack/events.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore/standalone/telemetry"
+)
+
+// LocalStackEventsAPI intercepts fault events and forwards them to LocalStack as error status callbacks.
+type LocalStackEventsAPI struct {
+	*telemetry.StandaloneEventsAPI
+	adapter   *LocalStackAdapter
+	requestID string
+	mu        sync.RWMutex
+}
+
+func NewLocalStackEventsAPI(adapter *LocalStackAdapter) *LocalStackEventsAPI {
+	return &LocalStackEventsAPI{
+		adapter:             adapter,
+		StandaloneEventsAPI: new(telemetry.StandaloneEventsAPI),
+	}
+}
+
+func (ev *LocalStackEventsAPI) SendFault(data interop.FaultData) error {
+	_ = ev.StandaloneEventsAPI.SendFault(data)
+
+	requestID := string(data.RequestID)
+	if data.RequestID == "" {
+		ev.mu.RLock()
+		requestID = ev.requestID
+		ev.mu.RUnlock()
+	}
+
+	resp := ErrorResponse{
+		ErrorMessage: fmt.Sprintf("RequestId: %s Error: %s", requestID, data.ErrorMessage),
+		ErrorType:    string(data.ErrorType),
+	}
+
+	payload, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+
+	return ev.adapter.SendStatus(Error, payload)
+}
+
+func (ev *LocalStackEventsAPI) SetCurrentRequestID(id interop.RequestID) {
+	ev.mu.Lock()
+	defer ev.mu.Unlock()
+	ev.requestID = string(id)
+	ev.StandaloneEventsAPI.SetCurrentRequestID(id)
+}

cmd/localstack/main.go

@@ -179,18 +179,36 @@ func main() {
 	localStackLogsEgressApi := NewLocalStackLogsEgressAPI(logCollector)
 	tracer := NewLocalStackTracer()
 
+	// Create LocalStack adapter upfront so it can be shared with the events API and interop server
+	lsAdapter := &LocalStackAdapter{
+		UpstreamEndpoint: lsOpts.RuntimeEndpoint,
+		RuntimeId:        lsOpts.RuntimeId,
+	}
+
+	// Events API forwards runtime fault events (unexpected exits) to LocalStack as error callbacks
+	lsEventsAPI := NewLocalStackEventsAPI(lsAdapter)
+
+	// Supervisor intercepts runtime process terminations and emits fault events via the events API
+	supervisorCtx, cancelSupervisor := context.WithCancel(context.Background())
+
+	localStackSupv := NewLocalStackSupervisor(supervisorCtx, lsEventsAPI)
+
 	// build sandbox
 	sandbox := rapidcore.
 		NewSandboxBuilder().
 		//SetTracer(tracer).
 		AddShutdownFunc(func() {
 			log.Debugln("Stopping file watcher")
 			cancelFileWatcher()
+			log.Debugln("Stopping supervisor")
+			cancelSupervisor()
 		}).
 		SetExtensionsFlag(true).
 		SetInitCachingFlag(true).
 		SetLogsEgressAPI(localStackLogsEgressApi).
-		SetTracer(tracer)
+		SetTracer(tracer).
+		SetEventsAPI(lsEventsAPI).
+		SetSupervisor(localStackSupv)
 
 	// Corresponds to the 'AWS_LAMBDA_RUNTIME_API' environment variable.
 	// We need to ensure the runtime server is up before the INIT phase,
@@ -211,7 +229,7 @@ func main() {
 	runDaemon(d) // async
 
 	defaultInterop := sandbox.DefaultInteropServer()
-	interopServer := NewCustomInteropServer(lsOpts, defaultInterop, logCollector)
+	interopServer := NewCustomInteropServer(lsOpts, lsAdapter, defaultInterop, logCollector)
 	sandbox.SetInteropServer(interopServer)
 	if len(handler) > 0 {
 		sandbox.SetHandler(handler)

cmd/localstack/supervisor.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync/atomic"
+
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/fatalerror"
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/supervisor"
+	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/supervisor/model"
+	"github.com/google/uuid"
+	log "github.com/sirupsen/logrus"
+)
+
+// LocalStackSupervisor wraps a ProcessSupervisor and intercepts runtime process termination events.
+// When a runtime process exits unexpectedly it sends a fault event via the EventsAPI so LocalStack
+// receives a proper error instead of timing out.
+type LocalStackSupervisor struct {
+	model.ProcessSupervisor
+	eventsChan chan model.Event
+	eventsAPI  interop.EventsAPI
+
+	isShuttingDown *atomic.Bool
+}
+
+func NewLocalStackSupervisor(ctx context.Context, evs interop.EventsAPI) *LocalStackSupervisor {
+	var isShuttingDown atomic.Bool
+	ls := &LocalStackSupervisor{
+		ProcessSupervisor: supervisor.NewLocalSupervisor(),
+		eventsAPI:         evs,
+		eventsChan:        make(chan model.Event),
+		isShuttingDown:    &isShuttingDown,
+	}
+
+	go ls.loop(ctx)
+
+	return ls
+}
+
+func (ls *LocalStackSupervisor) loop(ctx context.Context) {
+	inCh, err := ls.ProcessSupervisor.Events(ctx, nil)
+	if err != nil {
+		panic(err)
+	}
+	defer close(ls.eventsChan)
+	for {
+		select {
+		case event, ok := <-inCh:
+			if !ok {
+				return
+			}
+
+			select {
+			case ls.eventsChan <- event:
+			case <-ctx.Done():
+				return
+			}
+
+			if ls.isShuttingDown.Load() {
+				continue
+			}
+
+			termination := event.Event.ProcessTerminated()
+			if termination == nil {
+				continue
+			}
+
+			if !strings.Contains(*termination.Name, "runtime-") {
+				log.Debugf("Ignoring non-runtime process termination: %s", *termination.Name)
+				continue
+			}
+
+			if termination.Signaled() != nil {
+				log.Debugf("Runtime process signalled: %d", *termination.Signo)
+			}
+
+			faultData := interop.FaultData{
+				RequestID:    interop.RequestID(uuid.NewString()),
+				ErrorMessage: fmt.Errorf("Runtime exited without providing a reason"),
+				ErrorType:    fatalerror.RuntimeExit,
+			}
+			if !termination.Success() {
+				faultData.ErrorMessage = fmt.Errorf("Runtime exited with error: %s", termination.String())
+			}
+
+			if err := ls.eventsAPI.SendFault(faultData); err != nil {
+				log.WithError(err).Error("Failed to send runtime fault event")
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (ls *LocalStackSupervisor) Exec(ctx context.Context, request *model.ExecRequest) error {
+	if request.Domain == "runtime" {
+		ls.isShuttingDown.Store(false)
+	}
+	return ls.ProcessSupervisor.Exec(ctx, request)
+}
+
+func (ls *LocalStackSupervisor) Terminate(ctx context.Context, request *model.TerminateRequest) error {
+	defer func() {
+		if request.Domain == "runtime" && strings.HasPrefix(request.Name, "runtime-") {
+			ls.isShuttingDown.Store(true)
+		}
+	}()
+	return ls.ProcessSupervisor.Terminate(ctx, request)
+}
+
+func (ls *LocalStackSupervisor) Kill(ctx context.Context, request *model.KillRequest) error {
+	defer func() {
+		if request.Domain == "runtime" && strings.HasPrefix(request.Name, "runtime-") {
+			ls.isShuttingDown.Store(true)
+		}
+	}()
+	return ls.ProcessSupervisor.Kill(ctx, request)
+}
+
+func (ls *LocalStackSupervisor) Events(ctx context.Context, _ *model.EventsRequest) (<-chan model.Event, error) {
+	return ls.eventsChan, nil
+}