feat(all): implement idle process management for MCP servers

Author: Lasim

services/backend/src/events/satellite/index.ts

@@ -22,6 +22,9 @@ const handlerModules = [
   () => import('./mcp-tool-executed'),
   () => import('./mcp-server-crashed'),
   () => import('./mcp-client-activity'),
+  () => import('./mcp-server-dormant'),
+  () => import('./mcp-server-respawned'),
+  () => import('./mcp-server-restarted'),
   // Add new handlers here - they will be automatically registered
 ];
 

services/backend/src/events/satellite/mcp-server-dormant.ts

@@ -0,0 +1,86 @@
+/**
+ * MCP Server Dormant Event Handler
+ * 
+ * Updates satelliteProcesses table when an MCP server process goes dormant due to inactivity
+ */
+
+import type { LibSQLDatabase } from 'drizzle-orm/libsql';
+import { satelliteProcesses } from '../../db/schema.sqlite';
+import { eq } from 'drizzle-orm';
+
+// Event type identifier
+export const EVENT_TYPE = 'mcp.server.dormant';
+
+// JSON Schema for Fastify validation
+export const SCHEMA = {
+  type: 'object',
+  properties: {
+    server_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server identifier (installation_id)'
+    },
+    server_slug: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server slug (installation_name)'
+    },
+    team_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'Team identifier'
+    },
+    process_id: {
+      type: 'number',
+      description: 'Operating system process ID'
+    },
+    idle_duration_seconds: {
+      type: 'number',
+      minimum: 0,
+      description: 'Duration of inactivity before going dormant'
+    },
+    last_activity_at: {
+      type: 'string',
+      format: 'date-time',
+      description: 'Last activity timestamp'
+    }
+  },
+  required: ['server_id', 'server_slug', 'team_id', 'process_id', 'idle_duration_seconds', 'last_activity_at'],
+  additionalProperties: true
+} as const;
+
+// TypeScript interface for type safety
+interface ServerDormantData {
+  server_id: string;
+  server_slug: string;
+  team_id: string;
+  process_id: number;
+  idle_duration_seconds: number;
+  last_activity_at: string;
+}
+
+/**
+ * Handle mcp.server.dormant event
+ * 
+ * Updates the satelliteProcesses table to mark the process as terminated (dormant).
+ * This event is emitted when a satellite terminates an idle stdio MCP server to save resources.
+ */
+export async function handle(
+  satelliteId: string,
+  eventData: Record<string, unknown>,
+  db: LibSQLDatabase,
+  eventTimestamp: Date
+): Promise<void> {
+  const data = eventData as unknown as ServerDormantData;
+  
+  // Update process status to stopped (dormant state)
+  await db
+    .update(satelliteProcesses)
+    .set({
+      status: 'stopped',
+      health_status: 'unknown',
+      stopped_at: eventTimestamp,
+      updated_at: new Date()
+    })
+    .where(eq(satelliteProcesses.id, data.server_id));
+}

services/backend/src/events/satellite/mcp-server-respawned.ts

@@ -0,0 +1,87 @@
+/**
+ * MCP Server Respawned Event Handler
+ * 
+ * Updates satelliteProcesses table when a dormant MCP server is respawned
+ */
+
+import type { LibSQLDatabase } from 'drizzle-orm/libsql';
+import { satelliteProcesses } from '../../db/schema.sqlite';
+import { eq } from 'drizzle-orm';
+
+// Event type identifier
+export const EVENT_TYPE = 'mcp.server.respawned';
+
+// JSON Schema for Fastify validation
+export const SCHEMA = {
+  type: 'object',
+  properties: {
+    server_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server identifier (installation_id)'
+    },
+    server_slug: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server slug (installation_name)'
+    },
+    team_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'Team identifier'
+    },
+    process_id: {
+      type: 'number',
+      description: 'New operating system process ID'
+    },
+    dormant_duration_seconds: {
+      type: 'number',
+      minimum: 0,
+      description: 'Duration process was dormant'
+    },
+    respawn_duration_ms: {
+      type: 'number',
+      minimum: 0,
+      description: 'Time taken to respawn process in milliseconds'
+    }
+  },
+  required: ['server_id', 'server_slug', 'team_id', 'process_id', 'dormant_duration_seconds', 'respawn_duration_ms'],
+  additionalProperties: true
+} as const;
+
+// TypeScript interface for type safety
+interface ServerRespawnedData {
+  server_id: string;
+  server_slug: string;
+  team_id: string;
+  process_id: number;
+  dormant_duration_seconds: number;
+  respawn_duration_ms: number;
+}
+
+/**
+ * Handle mcp.server.respawned event
+ * 
+ * Updates the satelliteProcesses table to mark the process as running again.
+ * This event is emitted when a satellite automatically respawns a dormant stdio MCP server.
+ */
+export async function handle(
+  satelliteId: string,
+  eventData: Record<string, unknown>,
+  db: LibSQLDatabase,
+  eventTimestamp: Date
+): Promise<void> {
+  const data = eventData as unknown as ServerRespawnedData;
+  
+  // Update process status back to running with new PID
+  await db
+    .update(satelliteProcesses)
+    .set({
+      status: 'running',
+      process_pid: data.process_id,
+      health_status: 'healthy',
+      started_at: eventTimestamp,
+      updated_at: new Date()
+    })
+    .where(eq(satelliteProcesses.id, data.server_id));
+}

services/backend/src/events/satellite/mcp-server-restarted.ts

@@ -0,0 +1,94 @@
+/**
+ * MCP Server Restarted Event Handler
+ * 
+ * Updates satelliteProcesses table when an MCP server is restarted after a crash
+ */
+
+import type { LibSQLDatabase } from 'drizzle-orm/libsql';
+import { satelliteProcesses } from '../../db/schema.sqlite';
+import { eq } from 'drizzle-orm';
+
+// Event type identifier
+export const EVENT_TYPE = 'mcp.server.restarted';
+
+// JSON Schema for Fastify validation
+export const SCHEMA = {
+  type: 'object',
+  properties: {
+    server_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server identifier (installation_id)'
+    },
+    server_slug: {
+      type: 'string',
+      minLength: 1,
+      description: 'MCP server slug (installation_name)'
+    },
+    team_id: {
+      type: 'string',
+      minLength: 1,
+      description: 'Team identifier'
+    },
+    old_process_id: {
+      type: 'number',
+      description: 'Previous operating system process ID'
+    },
+    new_process_id: {
+      type: 'number',
+      description: 'New operating system process ID after restart'
+    },
+    restart_reason: {
+      type: 'string',
+      enum: ['crash', 'health_check_failed'],
+      description: 'Reason for the restart'
+    },
+    attempt_number: {
+      type: 'number',
+      minimum: 1,
+      maximum: 3,
+      description: 'Restart attempt number (1-3)'
+    }
+  },
+  required: ['server_id', 'server_slug', 'team_id', 'old_process_id', 'new_process_id', 'restart_reason', 'attempt_number'],
+  additionalProperties: true
+} as const;
+
+// TypeScript interface for type safety
+interface ServerRestartedData {
+  server_id: string;
+  server_slug: string;
+  team_id: string;
+  old_process_id: number;
+  new_process_id: number;
+  restart_reason: 'crash' | 'health_check_failed';
+  attempt_number: number;
+}
+
+/**
+ * Handle mcp.server.restarted event
+ * 
+ * Updates the satelliteProcesses table to mark the process as running again with new PID.
+ * This event is emitted when a satellite automatically restarts a crashed MCP server.
+ */
+export async function handle(
+  satelliteId: string,
+  eventData: Record<string, unknown>,
+  db: LibSQLDatabase,
+  eventTimestamp: Date
+): Promise<void> {
+  const data = eventData as unknown as ServerRestartedData;
+  
+  // Update process status back to running with new PID
+  await db
+    .update(satelliteProcesses)
+    .set({
+      status: 'running',
+      process_pid: data.new_process_id,
+      health_status: 'healthy',
+      started_at: eventTimestamp,
+      error_message: null, // Clear previous error
+      updated_at: new Date()
+    })
+    .where(eq(satelliteProcesses.id, data.server_id));
+}

services/satellite/.env.example

@@ -70,3 +70,10 @@ NSJAIL_CPU_TIME_LIMIT_SECONDS=60
 
 # Maximum number of processes per MCP server (default: 50)
 NSJAIL_MAX_PROCESSES=50
+
+# Process Idle Timeout (stdio MCP servers only)
+# Idle stdio processes are automatically terminated after this duration to save resources
+# Processes are transparently respawned when API calls arrive (1-3s latency)
+# Set to 0 to disable idle timeout (processes never sleep)
+# Default: 180 seconds (3 minutes)
+MCP_PROCESS_IDLE_TIMEOUT_SECONDS=180

services/satellite/README.md

@@ -110,6 +110,9 @@ EVENT_FLUSH_TIMEOUT_MS=5000         # Graceful shutdown flush timeout in millise
 NSJAIL_MEMORY_LIMIT_MB=50           # Memory limit per MCP server process in MB (default: 50)
 NSJAIL_CPU_TIME_LIMIT_SECONDS=60    # CPU time limit per MCP server process in seconds (default: 60)
 NSJAIL_MAX_PROCESSES=50             # Maximum number of processes per MCP server (default: 50)
+
+# Process Idle Timeout (stdio MCP servers only)
+MCP_PROCESS_IDLE_TIMEOUT_SECONDS=180 # Idle timeout in seconds before terminating stdio processes (default: 180, set to 0 to disable)
 ```
 
 ### nsjail Resource Limits
@@ -140,6 +143,54 @@ These limits control resource allocation for MCP server processes running in nsj
 - Allows easier debugging on macOS, Windows, and Linux
 - Full isolation only active in production Linux deployments
 
+### Process Idle Timeout (stdio MCP servers only)
+
+**MCP_PROCESS_IDLE_TIMEOUT_SECONDS** (Default: 180)
+
+Automatically terminates idle stdio MCP server processes to save resources. Dormant processes are transparently respawned when API calls arrive.
+
+**How It Works:**
+- Background job checks all stdio processes every 30 seconds
+- Processes idle longer than threshold are gracefully terminated
+- Process configurations stored in memory for automatic respawning
+- When API call arrives for dormant process, it respawns automatically (1-3s latency)
+
+**Benefits:**
+- **Memory Savings**: ~50-100MB per dormant process
+- **CPU Savings**: Zero overhead for idle processes
+- **Transparent**: MCP clients unaware of sleep/wake cycle
+- **Team Isolation**: Maintained (separate processes per team)
+
+**Configuration Options:**
+- `180` (default): 3 minutes idle timeout
+- `60`: 1 minute idle timeout (aggressive, for high-density deployments)
+- `600`: 10 minutes idle timeout (conservative, for frequently-used servers)
+- `0`: Disable idle timeout (processes never sleep)
+
+**Edge Cases Handled:**
+- Only terminates processes with `status=running`
+- Skips processes with active requests in flight
+- Prevents concurrent respawn attempts
+- Dormant processes excluded from heartbeat reports
+
+**Monitoring:**
+```bash
+# Check idle process activity
+grep "idle_process_check_completed" logs/satellite.log
+
+# Track dormant transitions
+grep "process_marked_dormant" logs/satellite.log
+
+# Monitor respawn operations
+grep "dormant_process_respawned" logs/satellite.log
+```
+
+**When to Adjust:**
+- **High-density deployments**: Lower timeout (60-120s) to maximize resource savings
+- **Frequently-used servers**: Higher timeout (300-600s) to avoid unnecessary respawns
+- **Development/testing**: Disable (0) to avoid respawn delays during debugging
+- **Production with ample resources**: Keep default (180s) for balanced operation
+
 ### Required Environment Variables
 
 **DEPLOYSTACK_SATELLITE_NAME** (Mandatory)

services/satellite/src/config/process.ts

@@ -0,0 +1,28 @@
+/**
+ * Process management configuration
+ */
+
+/**
+ * Idle timeout for stdio MCP server processes
+ * Processes inactive for longer than this will be terminated and marked dormant
+ * They will be automatically respawned when needed
+ * 
+ * Default: 180 seconds (3 minutes)
+ * Configure via: MCP_PROCESS_IDLE_TIMEOUT_SECONDS environment variable
+ */
+export const IDLE_TIMEOUT_MS = parseInt(
+  process.env.MCP_PROCESS_IDLE_TIMEOUT_SECONDS || '180',
+  10
+) * 1000;
+
+/**
+ * Grace period after process spawn during which it cannot be marked idle
+ * This prevents newly spawned processes from being terminated before they finish initialization
+ * 
+ * Default: 60 seconds
+ * Configure via: MCP_PROCESS_SPAWN_GRACE_PERIOD_SECONDS environment variable
+ */
+export const SPAWN_GRACE_PERIOD_MS = parseInt(
+  process.env.MCP_PROCESS_SPAWN_GRACE_PERIOD_SECONDS || '60',
+  10
+) * 1000;