From 7f939c8d499cd343d2bd88b2631d5de37a1845c9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 03:13:26 -0400 Subject: [PATCH 01/24] Experiment with lean CI realtime stream --- cli/XCWH264Encoder.m | 19 +++++++++++++++---- docs/guide/video.md | 2 +- server/src/api/routes.rs | 6 +++--- server/src/main.rs | 6 +++--- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index a2987b6..e0b02a2 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -143,10 +143,18 @@ static int64_t XCWRealtimeBitsPerPixelBudgetValue(void) { static int32_t XCWRealtimeMinimumAverageBitRate(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_MIN_BITRATE", XCWMinimumRealtimeAverageBitRate, - 750000, + 200000, 20000000); } +static double XCWRealtimeKeyFrameIntervalSeconds(void) { + NSString *value = NSProcessInfo.processInfo.environment[@"SIMDECK_REALTIME_KEYFRAME_INTERVAL_SECONDS"]; + if (value.length == 0) { + return 4.0; + } + return fmin(10.0, fmax(1.0, value.doubleValue)); +} + static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { switch (mode) { case XCWVideoEncoderModeH264Hardware: @@ -924,9 +932,12 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height VTSessionSetProperty(session, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel); } VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate)); - BOOL shortKeyframeInterval = _lowLatencyMode || _realtimeStreamMode; - VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? expectedFrameRate : expectedFrameRate * 2)); - VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 1.0 : 2.0)); + double keyframeIntervalSeconds = _realtimeStreamMode + ? XCWRealtimeKeyFrameIntervalSeconds() + : (_lowLatencyMode ? 1.0 : 2.0); + int keyframeIntervalFrames = MAX(1, (int)llround((double)expectedFrameRate * keyframeIntervalSeconds)); + VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(keyframeIntervalFrames)); + VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(keyframeIntervalSeconds)); VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate)); if (_realtimeStreamMode) { NSArray *dataRateLimits = @[ diff --git a/docs/guide/video.md b/docs/guide/video.md index 3503710..2d9ea5b 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -79,7 +79,7 @@ A few practical guidelines: - **Start on the default for compatibility.** `h264-software` works without requiring the hardware encoder, but full-resolution latency can be high. - **Switch to `h264` on local Apple Silicon when hardware encode is available.** Hardware H.264 gives the smoothest local preview with the least CPU. - **Switch to `h264-software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at an 844-pixel longest edge, targets 20 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. +- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at a 640-pixel longest edge, targets 15 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. - **Use `h264-software --low-latency` only when you need the older extra-conservative software profile.** It caps at 15 fps, uses a single pending frame, reduces the longest edge to 1170 pixels, and backs off before software encode latency turns into seconds of stream delay. ## Tuning with metrics diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 0e77e7b..f67d79a 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -106,9 +106,9 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ StreamQualityProfile { id: "ci-software", label: "CI Software", - max_edge: 844, - fps: 20, - min_bitrate: 800_000, + max_edge: 640, + fps: 15, + min_bitrate: 350_000, bits_per_pixel: 1, }, StreamQualityProfile { diff --git a/server/src/main.rs b/server/src/main.rs index be29139..6585941 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -652,9 +652,9 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "ci-software", - max_edge: 844, - fps: 20, - min_bitrate: 800_000, + max_edge: 640, + fps: 15, + min_bitrate: 350_000, bits_per_pixel: 1, }), _ => anyhow::bail!("Unknown stream quality profile `{profile}`."), From 64cfd8e1961fc4c9811f40b7d742aecb60a0c03c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 03:30:32 -0400 Subject: [PATCH 02/24] Honor lean CI realtime profile --- cli/XCWH264Encoder.m | 4 ++-- server/src/api/routes.rs | 12 ++++++------ server/src/transport/webrtc.rs | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index e0b02a2..004b06e 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -113,14 +113,14 @@ static int64_t XCWInt64FromEnvironment(NSString *name, int64_t fallback, int64_t static int32_t XCWRealtimeMaximumEncodedDimension(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_MAX_EDGE", XCWMaximumRealtimeHardwareEncodedDimension, - 720, + 320, XCWMaximumEncodedDimension); } static int32_t XCWRealtimeTargetFrameRate(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_FPS", XCWTargetRealtimeHardwareFrameRate, - 15, + 10, 60); } diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index f67d79a..675bdcb 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -591,17 +591,17 @@ async fn set_stream_quality( .max_edge .or_else(|| profile.map(|profile| profile.max_edge)) .unwrap_or(1440) - .clamp(720, 1920); + .clamp(320, 1920); let fps = payload .fps .or_else(|| profile.map(|profile| profile.fps)) .unwrap_or(30) - .clamp(15, 60); + .clamp(10, 60); let min_bitrate = payload .min_bitrate .or_else(|| profile.map(|profile| profile.min_bitrate)) .unwrap_or(3_000_000) - .clamp(750_000, 20_000_000); + .clamp(200_000, 20_000_000); let bits_per_pixel = payload .bits_per_pixel .or_else(|| profile.map(|profile| profile.bits_per_pixel)) @@ -645,12 +645,12 @@ fn stream_quality_state() -> Value { min_bitrate: 3_000_000, bits_per_pixel: 4, }); - let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 720, 1920); - let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 15, 60); + let max_edge = env_u32("SIMDECK_REALTIME_MAX_EDGE", fallback.max_edge, 320, 1920); + let fps = env_u32("SIMDECK_REALTIME_FPS", fallback.fps, 10, 60); let min_bitrate = env_u32( "SIMDECK_REALTIME_MIN_BITRATE", fallback.min_bitrate, - 750_000, + 200_000, 20_000_000, ); let bits_per_pixel = env_u32( diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 9ed96e7..bd6c317 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,8 +36,8 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(100); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); From f0e67fe6734901dbcf633b584fbea973e875b3a1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 11:53:06 -0400 Subject: [PATCH 03/24] Optimize CI streaming experiments --- cli/XCWH264Encoder.m | 169 +++++++++++++++++- .../src/features/stream/streamWorkerClient.ts | 148 ++++++++++++++- client/src/features/stream/useLiveStream.ts | 4 +- client/src/styles/components.css | 1 + server/src/api/routes.rs | 115 ++++++++++++ server/src/main.rs | 2 + server/src/transport/webrtc.rs | 12 +- 7 files changed, 437 insertions(+), 14 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 004b06e..3ee42f4 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -2,6 +2,7 @@ #import #import +#import #import #import #import @@ -40,6 +41,7 @@ typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, + XCWVideoEncoderModeJPEG, }; static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { @@ -51,6 +53,9 @@ static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { if ([value isEqualToString:@"h264-software"] || [value isEqualToString:@"software-h264"]) { return XCWVideoEncoderModeH264Software; } + if ([value isEqualToString:@"jpeg"] || [value isEqualToString:@"jpg"] || [value isEqualToString:@"mjpeg"]) { + return XCWVideoEncoderModeJPEG; + } return XCWVideoEncoderModeH264Software; } @@ -155,10 +160,104 @@ static double XCWRealtimeKeyFrameIntervalSeconds(void) { return fmin(10.0, fmax(1.0, value.doubleValue)); } +static CGFloat XCWJPEGQualityFromEnvironment(void) { + NSString *value = NSProcessInfo.processInfo.environment[@"SIMDECK_JPEG_QUALITY"]; + double quality = value.length > 0 ? value.doubleValue : 0.7; + if (!isfinite(quality)) { + return 0.7; + } + return (CGFloat)fmin(1.0, fmax(0.1, quality)); +} + +static CGColorSpaceRef XCWDeviceRGBColorSpace(void) { + static CGColorSpaceRef colorSpace = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + colorSpace = CGColorSpaceCreateDeviceRGB(); + }); + return colorSpace; +} + +static NSData *XCWJPEGDataFromPixelBuffer(CVPixelBufferRef pixelBuffer) { + if (pixelBuffer == NULL) { + return nil; + } + + CGImageRef image = NULL; + BOOL didLockPixelBuffer = NO; + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat == kCVPixelFormatType_32BGRA && + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) == kCVReturnSuccess) { + didLockPixelBuffer = YES; + void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); + if (baseAddress != NULL && width > 0 && height > 0 && bytesPerRow >= width * 4) { + CGDataProviderRef provider = CGDataProviderCreateWithData(NULL, + baseAddress, + bytesPerRow * height, + NULL); + if (provider != NULL) { + image = CGImageCreate(width, + height, + 8, + 32, + bytesPerRow, + XCWDeviceRGBColorSpace(), + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, + provider, + NULL, + false, + kCGRenderingIntentDefault); + CGDataProviderRelease(provider); + } + } + } + + if (image == NULL) { + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + didLockPixelBuffer = NO; + } + OSStatus imageStatus = VTCreateCGImageFromCVPixelBuffer(pixelBuffer, NULL, &image); + if (imageStatus != noErr || image == NULL) { + return nil; + } + } + + NSMutableData *data = [NSMutableData data]; + CGImageDestinationRef destination = + CGImageDestinationCreateWithData((__bridge CFMutableDataRef)data, + CFSTR("public.jpeg"), + 1, + NULL); + if (destination == NULL) { + CGImageRelease(image); + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + } + return nil; + } + + NSDictionary *properties = @{ + (__bridge NSString *)kCGImageDestinationLossyCompressionQuality: @(XCWJPEGQualityFromEnvironment()), + }; + CGImageDestinationAddImage(destination, image, (__bridge CFDictionaryRef)properties); + BOOL ok = CGImageDestinationFinalize(destination); + CFRelease(destination); + CGImageRelease(image); + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + } + return ok && data.length > 0 ? data : nil; +} + static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { switch (mode) { case XCWVideoEncoderModeH264Hardware: case XCWVideoEncoderModeH264Software: + case XCWVideoEncoderModeJPEG: default: return kCMVideoCodecType_H264; } @@ -169,6 +268,9 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { case XCWVideoEncoderModeH264Hardware: return @"h264"; case XCWVideoEncoderModeH264Software: + return @"h264-software"; + case XCWVideoEncoderModeJPEG: + return @"jpeg"; default: return @"h264-software"; } @@ -180,6 +282,8 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { return nil; case XCWVideoEncoderModeH264Software: return @"com.apple.videotoolbox.videoencoder.h264"; + case XCWVideoEncoderModeJPEG: + return nil; default: return nil; } @@ -691,6 +795,18 @@ - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { return YES; } +- (BOOL)shouldPaceJPEGFrameAtTimeUs:(uint64_t)nowUs { + if (_encoderMode != XCWVideoEncoderModeJPEG) { + return NO; + } + uint64_t intervalUs = XCWRealtimeFrameIntervalUs(); + if (_lastSoftwareSubmissionUs == 0) { + return NO; + } + uint64_t elapsedUs = nowUs >= _lastSoftwareSubmissionUs ? nowUs - _lastSoftwareSubmissionUs : 0; + return elapsedUs < intervalUs; +} + - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { if (_encoderMode != XCWVideoEncoderModeH264Software || latencyUs == 0) { return; @@ -805,10 +921,26 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { } uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); - if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs]) { + if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || + [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs] || + [self shouldPaceJPEGFrameAtTimeUs:nowUs]) { return YES; } + if (_encoderMode == XCWVideoEncoderModeJPEG) { + CVPixelBufferRef encodePixelBuffer = [self copyScaledPixelBufferIfNeeded:pixelBuffer + targetWidth:targetWidth + targetHeight:targetHeight]; + if (encodePixelBuffer == NULL) { + return NO; + } + BOOL ok = [self encodeJPEGPixelBufferLocked:encodePixelBuffer + sourceWidth:targetWidth + sourceHeight:targetHeight]; + CVPixelBufferRelease(encodePixelBuffer); + return ok; + } + if (![self ensureCompressionSessionWithWidth:targetWidth height:targetHeight]) { return NO; } @@ -861,6 +993,41 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { return YES; } +- (BOOL)encodeJPEGPixelBufferLocked:(CVPixelBufferRef)pixelBuffer + sourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight { + uint64_t submittedAtUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + if (_timestampOriginUs == 0) { + _timestampOriginUs = submittedAtUs; + } + uint64_t relativeTimestampUs = submittedAtUs - _timestampOriginUs; + + NSData *jpegData = XCWJPEGDataFromPixelBuffer(pixelBuffer); + if (jpegData.length == 0) { + _encodeFailureCount += 1; + _lastEncodeStatus = -1; + return NO; + } + + _width = sourceWidth; + _height = sourceHeight; + _lastSoftwareSubmissionUs = submittedAtUs; + _submittedFrameCount += 1; + _outputFrameCount += 1; + _keyFrameOutputCount += 1; + _lastEncodeStatus = noErr; + uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; + + self.outputHandler(jpegData, + relativeTimestampUs, + YES, + @"jpeg", + nil, + CGSizeMake(sourceWidth, sourceHeight)); + return YES; +} + - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height { if (_compressionSession != NULL && _width == width && _height == height) { return YES; diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 6cf4906..8a5a639 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -15,7 +15,7 @@ const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000; let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeStreamClient: StreamWorkerClient | null = null; -export type StreamBackend = "webrtc"; +export type StreamBackend = "mjpeg-img" | "webrtc"; export function sendWebRtcControlMessage(encoded: string): boolean { if (activeWebRtcControlChannel?.readyState !== "open") { @@ -41,6 +41,118 @@ interface StreamClientBackend { disconnect(): void; } +class MjpegImageStreamClient implements StreamClientBackend { + private canvas: HTMLCanvasElement | null = null; + private connectGeneration = 0; + private image: HTMLImageElement | null = null; + private statsInterval = 0; + private stats: StreamStats = createEmptyStreamStats(); + + constructor( + private readonly onMessage: (message: WorkerToMainMessage) => void, + ) {} + + attachCanvas(canvasElement: HTMLCanvasElement) { + this.canvas = canvasElement; + } + + clear() { + this.canvas + ?.getContext("2d") + ?.clearRect(0, 0, this.canvas.width, this.canvas.height); + } + + connect(target: StreamConnectTarget) { + this.disconnect(); + if (!this.canvas) { + return; + } + const canvasElement = this.canvas; + const generation = ++this.connectGeneration; + this.stats = { + ...createEmptyStreamStats(), + codec: "mjpeg", + }; + this.onMessage({ + type: "status", + status: { detail: "Opening MJPEG image stream", state: "connecting" }, + }); + + const image = document.createElement("img"); + image.alt = ""; + image.className = "stream-image"; + image.decoding = "async"; + image.draggable = false; + image.src = buildMjpegStreamUrl(target.udid); + image.addEventListener("load", () => { + if (generation !== this.connectGeneration) { + return; + } + this.reportImageSize(image); + this.stats.decodedFrames = Math.max(1, this.stats.decodedFrames); + this.stats.renderedFrames = Math.max(1, this.stats.renderedFrames); + this.stats.receivedPackets = Math.max(1, this.stats.receivedPackets); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + this.onMessage({ type: "status", status: { state: "streaming" } }); + }); + image.addEventListener("error", () => { + if (generation !== this.connectGeneration) { + return; + } + this.onMessage({ + type: "status", + status: { error: "MJPEG image stream failed.", state: "error" }, + }); + }); + canvasElement.after(image); + this.image = image; + this.statsInterval = window.setInterval(() => { + if (generation !== this.connectGeneration || !this.image) { + return; + } + this.reportImageSize(this.image); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + }, 1000); + } + + destroy() { + this.disconnect(); + } + + disconnect() { + this.connectGeneration += 1; + if (this.statsInterval) { + window.clearInterval(this.statsInterval); + this.statsInterval = 0; + } + if (this.image) { + this.image.removeAttribute("src"); + this.image.remove(); + this.image = null; + } + this.onMessage({ type: "status", status: { state: "idle" } }); + } + + private reportImageSize(image: HTMLImageElement) { + const width = image.naturalWidth; + const height = image.naturalHeight; + if (width <= 0 || height <= 0) { + return; + } + if (this.canvas) { + if (this.canvas.width !== width) { + this.canvas.width = width; + } + if (this.canvas.height !== height) { + this.canvas.height = height; + } + } + this.stats.width = width; + this.stats.height = height; + this.onMessage({ type: "video-config", size: { width, height } }); + } +} + class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; @@ -106,7 +218,8 @@ class WebRtcStreamClient implements StreamClientBackend { const controlChannel = peerConnection.createDataChannel( WEBRTC_CONTROL_CHANNEL_LABEL, { - ordered: true, + maxPacketLifeTime: 250, + ordered: false, }, ); this.controlChannel = controlChannel; @@ -740,6 +853,19 @@ function candidateStatsSummary(candidate: RTCStats | undefined): string { return `${stats.candidateType ?? "?"}/${stats.protocol ?? "?"}/${stats.address || stats.ip ? "addr" : "noaddr"}/${stats.port ?? "?"}`; } +function buildMjpegStreamUrl(udid: string): string { + const url = new URL( + `/api/simulators/${encodeURIComponent(udid)}/mjpeg`, + window.location.href, + ); + const token = new URL(window.location.href).searchParams.get("simdeckToken"); + if (token) { + url.searchParams.set("simdeckToken", token); + } + url.searchParams.set("cacheBust", String(Date.now())); + return url.toString(); +} + function waitForIceGathering(peerConnection: RTCPeerConnection) { if (peerConnection.iceGatheringState === "complete") { return Promise.resolve(); @@ -758,6 +884,7 @@ function waitForIceGathering(peerConnection: RTCPeerConnection) { export class StreamWorkerClient { private readonly onMessage: (message: WorkerToMainMessage) => void; private backend: StreamClientBackend | null = null; + private canvas: HTMLCanvasElement | null = null; private attachedCanvas = false; private disposed = false; @@ -774,13 +901,24 @@ export class StreamWorkerClient { return; } - this.backend = new WebRtcStreamClient(this.onMessage); - this.backend.attachCanvas(canvasElement); + this.canvas = canvasElement; this.attachedCanvas = true; } - connect(target: StreamConnectTarget) { + async connect(target: StreamConnectTarget) { try { + const health = await fetchHealth().catch(() => null); + const shouldUseMjpeg = health?.videoCodec === "jpeg"; + const isMjpegBackend = this.backend instanceof MjpegImageStreamClient; + if (!this.backend || shouldUseMjpeg !== isMjpegBackend) { + this.backend?.destroy(); + this.backend = shouldUseMjpeg + ? new MjpegImageStreamClient(this.onMessage) + : new WebRtcStreamClient(this.onMessage); + if (this.canvas) { + this.backend.attachCanvas(this.canvas); + } + } const result = this.backend?.connect(target); if (result && typeof result.catch === "function") { result.catch((error: unknown) => { diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index a07d923..c9ffdab 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -288,7 +288,7 @@ export function useLiveStream({ runtimeInfo, stats, status, - streamBackend: "webrtc", - streamCanvasKey: `webrtc-${streamCanvasRevision}`, + streamBackend: stats.codec === "mjpeg" ? "mjpeg-img" : "webrtc", + streamCanvasKey: `stream-${streamCanvasRevision}`, }; } diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 9f493ba..edf27b0 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1167,6 +1167,7 @@ pointer-events: none; } +.stream-image, .stream-video { position: absolute; inset: 0; diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 675bdcb..2aa0a6e 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -8,6 +8,7 @@ use crate::metrics::counters::{ClientStreamStats, Metrics}; use crate::native::bridge::{LogFilters, NativeBridge}; use crate::simulators::registry::SessionRegistry; use crate::simulators::session::SimulatorSession; +use crate::transport::packet::SharedFrame; use axum::body::Body; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; use axum::extract::{ConnectInfo, Path, Query, State}; @@ -16,13 +17,16 @@ use axum::middleware::{from_fn_with_state, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; +use bytes::Bytes; use futures::{SinkExt, StreamExt}; use serde::Deserialize; use serde_json::Map; use serde_json::{json as json_value, Value}; use std::collections::VecDeque; +use std::convert::Infallible; use std::env; use std::net::SocketAddr; +use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; @@ -425,6 +429,7 @@ pub fn router(state: AppState) -> Router { .route("/api/simulators/{udid}/touch", post(send_touch)) .route("/api/simulators/{udid}/control", get(control_socket)) .route("/api/simulators/{udid}/webrtc/offer", post(webrtc_offer)) + .route("/api/simulators/{udid}/mjpeg", get(mjpeg_stream)) .route( "/api/simulators/{udid}/touch-sequence", post(send_touch_sequence), @@ -564,6 +569,7 @@ fn normalize_video_codec(codec: &str) -> Option<&'static str> { match codec.trim().to_ascii_lowercase().as_str() { "h264" | "h264-hardware" | "avc" => Some("h264"), "h264-software" | "software-h264" => Some("h264-software"), + "jpeg" | "jpg" | "mjpeg" => Some("jpeg"), _ => None, } } @@ -1149,6 +1155,115 @@ async fn webrtc_offer( .map(Json) } +async fn mjpeg_stream( + State(state): State, + Path(udid): Path, +) -> Result { + let session = state.registry.get_or_create_async(&udid).await?; + session.ensure_started_async().await?; + session.request_refresh(); + let first_frame = session + .wait_for_keyframe(Duration::from_secs(3)) + .await + .ok_or_else(|| AppError::internal("timed out waiting for simulator JPEG frame"))?; + if first_frame.codec.as_deref() != Some("jpeg") { + return Err(AppError::bad_request( + "MJPEG stream requires starting simdeck with --video-codec jpeg", + )); + } + + let stream = futures::stream::unfold( + MjpegStreamState { + first_frame: Some(first_frame), + metrics: state.metrics.clone(), + receiver: session.subscribe(), + session, + ticker: tokio::time::interval(Duration::from_millis(67)), + }, + |mut state| async move { + state.ticker.tick().await; + state.session.request_refresh(); + let mut frame = match state.first_frame.take() { + Some(frame) => frame, + None => match state.receiver.recv().await { + Ok(frame) => frame, + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + match state.receiver.recv().await { + Ok(frame) => frame, + Err(_) => return None, + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => return None, + }, + }; + let mut stale_frames = 0u64; + loop { + match state.receiver.try_recv() { + Ok(next_frame) => { + stale_frames += 1; + frame = next_frame; + } + Err(tokio::sync::broadcast::error::TryRecvError::Lagged(skipped)) => { + stale_frames = stale_frames.saturating_add(skipped); + } + Err( + tokio::sync::broadcast::error::TryRecvError::Empty + | tokio::sync::broadcast::error::TryRecvError::Closed, + ) => break, + } + } + if stale_frames > 0 { + state + .metrics + .frames_dropped_server + .fetch_add(stale_frames, Ordering::Relaxed); + } + if frame.codec.as_deref() != Some("jpeg") { + return None; + } + + let payload = mjpeg_part(&frame); + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + Some((Ok::(Bytes::from(payload)), state)) + }, + ); + + Response::builder() + .header( + header::CONTENT_TYPE, + "multipart/x-mixed-replace; boundary=simdeck-mjpeg", + ) + .header(header::CACHE_CONTROL, "no-store, no-transform") + .header("x-accel-buffering", "no") + .body(Body::from_stream(stream)) + .map_err(|error| AppError::internal(format!("failed to build MJPEG response: {error}"))) +} + +struct MjpegStreamState { + first_frame: Option, + metrics: Arc, + receiver: tokio::sync::broadcast::Receiver, + session: SimulatorSession, + ticker: tokio::time::Interval, +} + +fn mjpeg_part(frame: &SharedFrame) -> Vec { + let header = format!( + "--simdeck-mjpeg\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\nX-SimDeck-Frame: {}\r\n\r\n", + frame.data.len(), + frame.frame_sequence + ); + let mut payload = Vec::with_capacity(header.len() + frame.data.len() + 2); + payload.extend_from_slice(header.as_bytes()); + payload.extend_from_slice(&frame.data); + payload.extend_from_slice(b"\r\n"); + payload +} + async fn handle_control_socket(state: AppState, udid: String, socket: WebSocket) { let session = match state.registry.get_or_create_async(&udid).await { Ok(session) => session, diff --git a/server/src/main.rs b/server/src/main.rs index 6585941..44db4d0 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -519,6 +519,7 @@ enum PasteboardCommand { enum VideoCodecMode { H264, H264Software, + Jpeg, } #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] @@ -608,6 +609,7 @@ impl VideoCodecMode { match self { Self::H264 => "h264", Self::H264Software => "h264-software", + Self::Jpeg => "jpeg", } } } diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index bd6c317..78a5cd2 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -39,10 +39,10 @@ const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); -const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(25); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(60); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); @@ -852,7 +852,7 @@ async fn write_frame_sample_with_timeout( if frame.is_keyframe { WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT } else { - WEBRTC_REALTIME_WRITE_TIMEOUT.max(realtime_sample_duration() * 2) + WEBRTC_REALTIME_WRITE_TIMEOUT } } else { WEBRTC_WRITE_TIMEOUT @@ -873,7 +873,7 @@ fn realtime_packet_pacing( packet_count: usize, realtime_stream: bool, ) -> Option<(usize, Duration)> { - if !realtime_stream || packet_count <= 1 { + if realtime_stream || packet_count <= 1 { return None; } let pacing_ticks = ((duration.as_millis() / 4).max(1) as usize).min(packet_count - 1); @@ -1040,7 +1040,7 @@ fn realtime_sample_duration() -> Duration { .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(30) - .clamp(15, 60); + .clamp(30, 60); Duration::from_micros(1_000_000 / fps) } From 3da65198da7b0eb52b0eae5f6d66670ce709d63b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 12:31:42 -0400 Subject: [PATCH 04/24] Restore usable CI software stream quality --- server/src/api/routes.rs | 8 ++++---- server/src/transport/webrtc.rs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 2aa0a6e..b92af05 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -110,10 +110,10 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ StreamQualityProfile { id: "ci-software", label: "CI Software", - max_edge: 640, - fps: 15, - min_bitrate: 350_000, - bits_per_pixel: 1, + max_edge: 960, + fps: 24, + min_bitrate: 1_200_000, + bits_per_pixel: 2, }, StreamQualityProfile { id: "quality", diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 78a5cd2..7ed04c6 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -41,8 +41,8 @@ const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(25); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(60); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); @@ -1040,7 +1040,7 @@ fn realtime_sample_duration() -> Duration { .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(30) - .clamp(30, 60); + .clamp(15, 60); Duration::from_micros(1_000_000 / fps) } From 51988cf9df56be465a12fe5830151f98b547f305 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 13:12:27 -0400 Subject: [PATCH 05/24] Retry CoreSimulator headless screen attach --- cli/DFPrivateSimulatorDisplayBridge.m | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 7794843..82ba047 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -2429,9 +2429,30 @@ - (nullable instancetype)initWithUDID:(NSString *)udid DFLog(@"Initialized bootstrap SimDeviceScreen for %@", udid); } [self updateStatus:@"Waiting for CoreSimulator screen adapter"]; - DFSpinRunLoop(0.5); + NSDate *adapterDeadline = [NSDate dateWithTimeIntervalSinceNow:15.0]; + while (_screenAdapter == nil && [adapterDeadline timeIntervalSinceNow] > 0) { + Ivar screenAdapterIvar = class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter"); + if (screenAdapterIvar != NULL) { + _screenAdapter = object_getIvar(_screenAdapterHost, screenAdapterIvar); + } + if (_screenAdapter != nil) { + break; + } - _screenAdapter = object_getIvar(_screenAdapterHost, class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter")); + // On cold CoreSimulator boots the Swift screenAdapter host can exist + // before its private ROCK proxy is connected. Re-read the extension + // property while spinning the run loop instead of failing the attach. + id refreshedHost = DFCallSwiftSelfGetterByPattern( + _device, + "$sSo9SimDeviceC12SimulatorKitE13screenAdapter", + "vg", + "SimDevice.screenAdapter.getter (SimulatorKit extension)" + ); + if (refreshedHost != nil) { + _screenAdapterHost = refreshedHost; + } + DFSpinRunLoop(0.2); + } if (_screenAdapter == nil) { DFLog(@"SimulatorKit screen adapter host did not expose _screenAdapter for %@; host class=%@", udid, NSStringFromClass([_screenAdapterHost class])); if (error != NULL) { @@ -2475,7 +2496,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid // seconds (or only after Simulator.app primes the device). Poll instead of // relying on a fixed 0.5s sleep. NSDictionary *screens = DFReadAvailableAdapterScreens(_screenAdapterHost, _screenAdapter); - NSDate *screenDeadline = [NSDate dateWithTimeIntervalSinceNow:10.0]; + NSDate *screenDeadline = [NSDate dateWithTimeIntervalSinceNow:20.0]; while (screens.count == 0 && [screenDeadline timeIntervalSinceNow] > 0) { DFSpinRunLoop(0.1); screens = DFReadAvailableAdapterScreens(_screenAdapterHost, _screenAdapter); From 4e4f0b7d2f626a4f908ff38b8820d966566f2611 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 13:22:29 -0400 Subject: [PATCH 06/24] Handle direct SimulatorKit screen adapters --- cli/DFPrivateSimulatorDisplayBridge.m | 84 ++++++++++++++++++--------- server/src/main.rs | 8 +-- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 82ba047..683c355 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -617,16 +617,35 @@ static id DFInitSimDeviceScreen(Class screenClass, id device, uint32_t screenID, return @{}; } + SEL screensSelector = sel_registerName("screens"); + id screens = nil; + if ([adapter respondsToSelector:screensSelector]) { + @try { + screens = ((id(*)(id, SEL))objc_msgSend)(adapter, screensSelector); + } @catch (NSException *exception) { + DFLog(@"SimulatorKit screen adapter screens selector threw: %@", exception.reason ?: exception.name); + screens = nil; + } + } + if (screens == nil) { + Ivar screensIvar = class_getInstanceVariable([adapter class], "_screens"); + if (screensIvar != NULL) { + screens = object_getIvar(adapter, screensIvar); + } + } + // The full mangled tail of `SimDeviceScreenAdapter.screens.getter` drifts // across Xcode releases (Xcode 26.4 retyped it from // `[UInt32: SimScreen]` (ObjC) to `[UInt32: SimDeviceScreen]` (Swift)). // Resolve by stable prefix instead. If the values are now SimDeviceScreen // wrappers, unwrap each via `.screen` so callers keep talking to a // SimScreen-shaped object. - id screens = DFCallSwiftSelfGetter( - adapter, - "$s12SimulatorKit22SimDeviceScreenAdapterC7screensSDys6UInt32VSo0cE0_pGvg" - ); + if (screens == nil) { + screens = DFCallSwiftSelfGetter( + adapter, + "$s12SimulatorKit22SimDeviceScreenAdapterC7screensSDys6UInt32VSo0cE0_pGvg" + ); + } if (screens == nil) { screens = DFCallSwiftSelfGetter( adapter, @@ -2431,9 +2450,13 @@ - (nullable instancetype)initWithUDID:(NSString *)udid [self updateStatus:@"Waiting for CoreSimulator screen adapter"]; NSDate *adapterDeadline = [NSDate dateWithTimeIntervalSinceNow:15.0]; while (_screenAdapter == nil && [adapterDeadline timeIntervalSinceNow] > 0) { - Ivar screenAdapterIvar = class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter"); - if (screenAdapterIvar != NULL) { - _screenAdapter = object_getIvar(_screenAdapterHost, screenAdapterIvar); + if ([NSStringFromClass([_screenAdapterHost class]) containsString:@"SimDeviceScreenAdapter"]) { + _screenAdapter = _screenAdapterHost; + } else { + Ivar screenAdapterIvar = class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter"); + if (screenAdapterIvar != NULL) { + _screenAdapter = object_getIvar(_screenAdapterHost, screenAdapterIvar); + } } if (_screenAdapter != nil) { break; @@ -2466,28 +2489,33 @@ - (nullable instancetype)initWithUDID:(NSString *)udid _screenAdapterCallbackUUID = [NSUUID UUID]; __weak typeof(self) weakSelf = self; - ((void(*)(id, SEL, id, id, id, id))objc_msgSend)( - _screenAdapter, - sel_registerName("registerScreenAdapterCallbacksWithUUID:callbackQueue:screenConnectedCallback:screenWillDisconnectCallback:"), - _screenAdapterCallbackUUID, - _callbackQueue, - ^(id simScreen) { - (void)simScreen; - __strong typeof(weakSelf) strongSelf = weakSelf; - if (strongSelf == nil) { - return; - } - [strongSelf updateStatus:@"CoreSimulator screen proxy connected"]; - }, - ^(id simScreen) { - (void)simScreen; - __strong typeof(weakSelf) strongSelf = weakSelf; - if (strongSelf == nil) { - return; + SEL registerCallbacksSelector = sel_registerName("registerScreenAdapterCallbacksWithUUID:callbackQueue:screenConnectedCallback:screenWillDisconnectCallback:"); + if ([_screenAdapter respondsToSelector:registerCallbacksSelector]) { + ((void(*)(id, SEL, id, id, id, id))objc_msgSend)( + _screenAdapter, + registerCallbacksSelector, + _screenAdapterCallbackUUID, + _callbackQueue, + ^(id simScreen) { + (void)simScreen; + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil) { + return; + } + [strongSelf updateStatus:@"CoreSimulator screen proxy connected"]; + }, + ^(id simScreen) { + (void)simScreen; + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil) { + return; + } + [strongSelf updateStatus:@"CoreSimulator screen proxy disconnected"]; } - [strongSelf updateStatus:@"CoreSimulator screen proxy disconnected"]; - } - ); + ); + } else { + DFLog(@"SimulatorKit screen adapter %@ does not expose callback registration; polling screens directly", NSStringFromClass([_screenAdapter class])); + } [self updateStatus:@"Waiting for headless simulator screens"]; diff --git a/server/src/main.rs b/server/src/main.rs index 44db4d0..65e1b4a 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -654,10 +654,10 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "ci-software", - max_edge: 640, - fps: 15, - min_bitrate: 350_000, - bits_per_pixel: 1, + max_edge: 960, + fps: 24, + min_bitrate: 1_200_000, + bits_per_pixel: 2, }), _ => anyhow::bail!("Unknown stream quality profile `{profile}`."), } From fa9f37be923f534dd03180453f48b9a3a101ef53 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 13:25:46 -0400 Subject: [PATCH 07/24] Restore local hardware H264 defaults --- client/src/app/AppShell.tsx | 38 +++++++++++++++++++++++++++++++++++++ server/src/main.rs | 18 +++++++++--------- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index c20b464..daa6602 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -190,6 +190,9 @@ export function AppShell() { ); const [menuOpen, setMenuOpen] = useState(false); const [localError, setLocalError] = useState(""); + const [failedStreamUDIDs, setFailedStreamUDIDs] = useState>( + () => new Set(), + ); const [pairingCode, setPairingCode] = useState(""); const [pairingError, setPairingError] = useState(""); const [pairingBusy, setPairingBusy] = useState(false); @@ -333,6 +336,31 @@ export function AppShell() { canvasElement: streamCanvasElement, simulator: selectedSimulator, }); + + useEffect(() => { + if ( + !selectedSimulator || + !streamError || + readDeviceQueryParam() || + !isStreamAttachFailure(streamError) + ) { + return; + } + const failedUDID = selectedSimulator.udid; + setFailedStreamUDIDs((current) => new Set(current).add(failedUDID)); + const nextSimulator = simulators.find( + (simulator) => + simulator.isBooted && + simulator.udid !== failedUDID && + !failedStreamUDIDs.has(simulator.udid), + ); + if (nextSimulator) { + setSelectedUDID(nextSimulator.udid); + setLocalError( + `${selectedSimulator.name} did not expose a live simulator screen. Switched to ${nextSimulator.name}.`, + ); + } + }, [failedStreamUDIDs, selectedSimulator, simulators, streamError]); const shouldRenderChrome = selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator); const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null; @@ -1389,3 +1417,13 @@ function readDeviceQueryParam(): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; } + +function isStreamAttachFailure(message: string): boolean { + const normalized = message.toLowerCase(); + return ( + normalized.includes("headless screen") || + normalized.includes("screen adapter") || + normalized.includes("coresimulator did not provide") || + normalized.includes("did not expose any live screens") + ); +} diff --git a/server/src/main.rs b/server/src/main.rs index 65e1b4a..3b97107 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -79,7 +79,7 @@ enum Command { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -106,7 +106,7 @@ enum Command { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -380,7 +380,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -396,7 +396,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -420,7 +420,7 @@ enum DaemonCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -463,7 +463,7 @@ enum ServiceCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -481,7 +481,7 @@ enum ServiceCommand { advertise_host: Option, #[arg(long)] client_root: Option, - #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] + #[arg(long, value_enum, default_value_t = VideoCodecMode::H264)] video_codec: VideoCodecMode, #[arg(long)] low_latency: bool, @@ -712,7 +712,7 @@ impl Default for DaemonLaunchOptions { bind: IpAddr::V4(Ipv4Addr::LOCALHOST), advertise_host: None, client_root: None, - video_codec: VideoCodecMode::H264Software, + video_codec: VideoCodecMode::H264, low_latency: false, realtime_stream: false, stream_quality_profile: None, @@ -1188,7 +1188,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let project_root = project_root()?; let port = choose_daemon_port(4310)?; let bind = IpAddr::V4(Ipv4Addr::UNSPECIFIED); - let video_codec = VideoCodecMode::H264Software; + let video_codec = VideoCodecMode::H264; let low_latency = false; let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) From 66b882707adb8c5c996dc99e3f228240691f8182 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 13:29:34 -0400 Subject: [PATCH 08/24] Probe foreground bind address for port selection --- server/src/main.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index 3b97107..f7a3cf3 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1073,17 +1073,26 @@ fn project_root() -> anyhow::Result { } fn choose_daemon_port(preferred: u16) -> anyhow::Result { + choose_daemon_port_for_bind(preferred, IpAddr::V4(Ipv4Addr::LOCALHOST)) +} + +fn choose_daemon_port_for_bind(preferred: u16, bind: IpAddr) -> anyhow::Result { let start = preferred.max(1024); for port in start..start.saturating_add(200) { - if port_available(port) { + if port_available(bind, port) { return Ok(port); } } anyhow::bail!("No available SimDeck daemon port near {preferred}") } -fn port_available(port: u16) -> bool { - TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_ok() +fn port_available(bind: IpAddr, port: u16) -> bool { + if bind.is_unspecified() { + if TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_err() { + return false; + } + } + TcpListener::bind((bind, port)).is_ok() } fn open_browser(url: &str) -> anyhow::Result<()> { @@ -1186,8 +1195,8 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { } let project_root = project_root()?; - let port = choose_daemon_port(4310)?; let bind = IpAddr::V4(Ipv4Addr::UNSPECIFIED); + let port = choose_daemon_port_for_bind(4310, bind)?; let video_codec = VideoCodecMode::H264; let low_latency = false; let advertise_host = detect_lan_ip() From a721e989c4b0fd487574258a90890014272620ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 13:46:16 -0400 Subject: [PATCH 09/24] Avoid loading loops on failed simulator streams --- client/src/app/AppShell.tsx | 12 ++++++++++++ client/src/features/stream/streamWorkerClient.ts | 15 +++++++++++++++ server/src/main.rs | 4 ++-- server/src/simulators/session.rs | 6 ++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index daa6602..2e03c88 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -135,6 +135,16 @@ function simulatorDisplaySize( }; } +function simulatorDisplayReady(simulator: SimulatorMetadata): boolean { + const display = simulator.privateDisplay; + return Boolean( + simulator.isBooted && + display?.displayReady && + display.displayWidth > 0 && + display.displayHeight > 0, + ); +} + function mergeAccessibilitySources( ...sources: unknown[] ): AccessibilitySource[] { @@ -295,6 +305,8 @@ export function AppShell() { simulators.find((simulator) => simulatorMatchesIdentifier(simulator, selectedUDID), ) ?? + filteredSimulators.find((simulator) => simulatorDisplayReady(simulator)) ?? + filteredSimulators.find((simulator) => simulator.isBooted) ?? filteredSimulators[0] ?? null; const selectedSimulatorDetail = diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 8a5a639..1acb980 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -381,6 +381,10 @@ class WebRtcStreamClient implements StreamClientBackend { type: "status", status: { error: message, state: "error" }, }); + if (isPermanentStreamFailure(message)) { + this.shouldReconnect = false; + return; + } this.scheduleReconnect(target, generation); } @@ -866,6 +870,17 @@ function buildMjpegStreamUrl(udid: string): string { return url.toString(); } +function isPermanentStreamFailure(message: string): boolean { + const normalized = message.toLowerCase(); + return ( + normalized.includes("private simulator display attach previously failed") || + normalized.includes("coresimulator did not provide") || + normalized.includes("did not expose any live screens") || + normalized.includes("headless screen") || + normalized.includes("screen adapter") + ); +} + function waitForIceGathering(peerConnection: RTCPeerConnection) { if (peerConnection.iceGatheringState === "complete") { return Promise.resolve(); diff --git a/server/src/main.rs b/server/src/main.rs index f7a3cf3..db63894 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -47,8 +47,8 @@ const SERVER_FD_RESTART_THRESHOLD: usize = 4096; const SERVER_HEALTH_WATCHDOG_INITIAL_DELAY: Duration = Duration::from_secs(15); const SERVER_HEALTH_WATCHDOG_INTERVAL: Duration = Duration::from_secs(5); const SERVER_HEALTH_WATCHDOG_PROBE_TIMEOUT: Duration = Duration::from_secs(3); -const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(10); -const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 3; +const SERVER_HEALTH_WATCHDOG_STALE_HEARTBEAT: Duration = Duration::from_secs(60); +const SERVER_HEALTH_WATCHDOG_FAILURE_THRESHOLD: usize = 12; #[derive(Parser)] #[command(name = "simdeck")] diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index cc7d1e4..a103ecc 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -80,6 +80,12 @@ impl SimulatorSession { if matches!(*state, SessionState::Ready | SessionState::Streaming) { return Ok(()); } + if matches!(*state, SessionState::Failed) { + return Err(AppError::native(format!( + "Private simulator display attach previously failed for {}. Restart the simulator or daemon before retrying this UDID.", + self.inner.udid + ))); + } *state = SessionState::Attaching; } From 229bfb5f7dbc3eb41cc735f79bae053d0406323f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:01:24 -0400 Subject: [PATCH 10/24] Restore smooth local WebRTC defaults --- cli/XCWH264Encoder.m | 64 +++++++++---------- .../src/features/stream/streamWorkerClient.ts | 16 ++++- server/src/api/routes.rs | 2 +- server/src/main.rs | 13 ++-- server/src/transport/webrtc.rs | 10 +-- 5 files changed, 58 insertions(+), 47 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 3ee42f4..0e55a79 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -35,9 +35,6 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; -static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; -static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; - typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, @@ -746,6 +743,26 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { return _lowLatencyMode ? XCWLowLatencySoftwareHealthyFrameWindow : XCWSoftwareHealthyFrameWindow; } +- (BOOL)fallbackToSoftwareEncoderIfHardwareUnavailableLocked { + if (_encoderMode != XCWVideoEncoderModeH264Hardware) { + return NO; + } + + _encoderMode = XCWVideoEncoderModeH264Software; + _lowLatencyMode = XCWLowLatencyModeFromEnvironment(); + _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; + _codecType = XCWVideoCodecTypeForMode(_encoderMode); + _hardwareAccelerated = NO; + _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + _lastSoftwareSubmissionUs = 0; + _lastRealtimeHardwareSubmissionUs = 0; + _softwareHealthyFrameCount = 0; + _realtimeHardwareHealthyFrameCount = 0; + _needsKeyFrame = YES; + return YES; +} + - (int32_t)expectedFrameRateLocked { if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { @@ -850,36 +867,7 @@ - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { return; } - if (_realtimeHardwareFrameIntervalUs == 0) { - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - } - - uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); - uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); - if (latencyUs > _realtimeHardwareFrameIntervalUs) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; - uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; - if (nextIntervalUs < latencyBoundIntervalUs) { - nextIntervalUs = latencyBoundIntervalUs; - } - _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - return; - } - - if (latencyUs < _realtimeHardwareFrameIntervalUs && - _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { - _realtimeHardwareHealthyFrameCount += 1; - if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs - ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs - : minimumIntervalUs; - _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - } - return; - } - + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); _realtimeHardwareHealthyFrameCount = 0; } @@ -1064,6 +1052,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height &session); _lastSessionStatus = status; if (status != noErr || session == NULL) { + if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { + return NO; + } return NO; } @@ -1131,9 +1122,14 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height _lastPrepareStatus = status; if (status != noErr) { [self invalidateCompressionSessionLocked]; + if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { + return NO; + } return NO; } - _hardwareAccelerated = XCWCompressionSessionUsesHardwareEncoder(session); + _hardwareAccelerated = (_encoderMode == XCWVideoEncoderModeH264Hardware) + ? YES + : XCWCompressionSessionUsesHardwareEncoder(session); return YES; } diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 1acb980..252058a 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -10,6 +10,7 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000; +const WEBRTC_STATS_REPORT_INTERVAL_MS = 250; const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000; let activeWebRtcControlChannel: RTCDataChannel | null = null; @@ -160,6 +161,7 @@ class WebRtcStreamClient implements StreamClientBackend { private controlChannel: RTCDataChannel | null = null; private diagnostics = createWebRtcDiagnostics(); private frameWatchdogTimeout = 0; + private lastStatsReportAt = 0; private lastVideoFrameAt = 0; private peerConnection: RTCPeerConnection | null = null; private reconnectTimeout = 0; @@ -192,6 +194,7 @@ class WebRtcStreamClient implements StreamClientBackend { const generation = ++this.connectGeneration; this.shouldReconnect = true; this.diagnostics = createWebRtcDiagnostics(); + this.lastStatsReportAt = 0; this.reportedVideoConfig = false; this.stats = createEmptyStreamStats(); this.onMessage({ @@ -613,11 +616,22 @@ class WebRtcStreamClient implements StreamClientBackend { this.stats.latestFrameGapMs = now - this.lastVideoFrameAt; } this.lastVideoFrameAt = now; - this.onMessage({ type: "stats", stats: { ...this.stats } }); + this.reportStats(now); } this.scheduleVideoFrame(); }; + private reportStats(now = performance.now()) { + if ( + this.lastStatsReportAt > 0 && + now - this.lastStatsReportAt < WEBRTC_STATS_REPORT_INTERVAL_MS + ) { + return; + } + this.lastStatsReportAt = now; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + } + private scheduleVideoFrame() { this.cancelVideoFrameCallback(); if (!this.video) { diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index b92af05..0f306e4 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -119,7 +119,7 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ id: "quality", label: "Quality", max_edge: 1440, - fps: 30, + fps: 60, min_bitrate: 3_000_000, bits_per_pixel: 4, }, diff --git a/server/src/main.rs b/server/src/main.rs index db63894..0792207 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -627,7 +627,7 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "quality", max_edge: 1440, - fps: 30, + fps: 60, min_bitrate: 3_000_000, bits_per_pixel: 4, }), @@ -714,8 +714,8 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: false, - stream_quality_profile: None, + realtime_stream: true, + stream_quality_profile: Some("quality".to_owned()), } } } @@ -1199,6 +1199,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let port = choose_daemon_port_for_bind(4310, bind)?; let video_codec = VideoCodecMode::H264; let low_latency = false; + let stream_quality_profile = Some("quality".to_owned()); let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) .to_string(); @@ -1216,8 +1217,8 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: false, - stream_quality_profile: None, + realtime_stream: true, + stream_quality_profile: stream_quality_profile.clone(), }; write_daemon_metadata(&metadata)?; @@ -1239,7 +1240,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { None, video_codec, low_latency, - None, + stream_quality_profile, Some(access_token), Some(pairing_code), ); diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 7ed04c6..8482b62 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,13 +36,13 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(120); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(180); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); @@ -873,7 +873,7 @@ fn realtime_packet_pacing( packet_count: usize, realtime_stream: bool, ) -> Option<(usize, Duration)> { - if realtime_stream || packet_count <= 1 { + if !realtime_stream || packet_count <= 1 { return None; } let pacing_ticks = ((duration.as_millis() / 4).max(1) as usize).min(packet_count - 1); From 1824195815ad8288a78ee7279f0c678fd518bbfb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:04:35 -0400 Subject: [PATCH 11/24] Reduce local WebRTC stream pressure --- server/src/transport/webrtc.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 8482b62..a834b9d 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,17 +36,17 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(67); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(180); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); -const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 3; +const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 1; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] From 41552f9100a65d72f82a66bbf2c54df6fccdca06 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:15:21 -0400 Subject: [PATCH 12/24] Revert "Reduce local WebRTC stream pressure" This reverts commit 1824195815ad8288a78ee7279f0c678fd518bbfb. --- server/src/transport/webrtc.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index a834b9d..8482b62 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,17 +36,17 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(120); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(180); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); -const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 1; +const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 3; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] From 2f1d4a7a0514260ee1e3dd9643807963f240c8fd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:15:22 -0400 Subject: [PATCH 13/24] Revert "Restore smooth local WebRTC defaults" This reverts commit 229bfb5f7dbc3eb41cc735f79bae053d0406323f. --- cli/XCWH264Encoder.m | 64 ++++++++++--------- .../src/features/stream/streamWorkerClient.ts | 16 +---- server/src/api/routes.rs | 2 +- server/src/main.rs | 13 ++-- server/src/transport/webrtc.rs | 10 +-- 5 files changed, 47 insertions(+), 58 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 0e55a79..3ee42f4 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -35,6 +35,9 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; +static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; +static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; + typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, @@ -743,26 +746,6 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { return _lowLatencyMode ? XCWLowLatencySoftwareHealthyFrameWindow : XCWSoftwareHealthyFrameWindow; } -- (BOOL)fallbackToSoftwareEncoderIfHardwareUnavailableLocked { - if (_encoderMode != XCWVideoEncoderModeH264Hardware) { - return NO; - } - - _encoderMode = XCWVideoEncoderModeH264Software; - _lowLatencyMode = XCWLowLatencyModeFromEnvironment(); - _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; - _codecType = XCWVideoCodecTypeForMode(_encoderMode); - _hardwareAccelerated = NO; - _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - _lastSoftwareSubmissionUs = 0; - _lastRealtimeHardwareSubmissionUs = 0; - _softwareHealthyFrameCount = 0; - _realtimeHardwareHealthyFrameCount = 0; - _needsKeyFrame = YES; - return YES; -} - - (int32_t)expectedFrameRateLocked { if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { @@ -867,7 +850,36 @@ - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { return; } - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + if (_realtimeHardwareFrameIntervalUs == 0) { + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + } + + uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); + uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); + if (latencyUs > _realtimeHardwareFrameIntervalUs) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; + uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; + if (nextIntervalUs < latencyBoundIntervalUs) { + nextIntervalUs = latencyBoundIntervalUs; + } + _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + return; + } + + if (latencyUs < _realtimeHardwareFrameIntervalUs && + _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { + _realtimeHardwareHealthyFrameCount += 1; + if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs + ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs + : minimumIntervalUs; + _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + } + return; + } + _realtimeHardwareHealthyFrameCount = 0; } @@ -1052,9 +1064,6 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height &session); _lastSessionStatus = status; if (status != noErr || session == NULL) { - if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { - return NO; - } return NO; } @@ -1122,14 +1131,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height _lastPrepareStatus = status; if (status != noErr) { [self invalidateCompressionSessionLocked]; - if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { - return NO; - } return NO; } - _hardwareAccelerated = (_encoderMode == XCWVideoEncoderModeH264Hardware) - ? YES - : XCWCompressionSessionUsesHardwareEncoder(session); + _hardwareAccelerated = XCWCompressionSessionUsesHardwareEncoder(session); return YES; } diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 252058a..1acb980 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -10,7 +10,6 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000; -const WEBRTC_STATS_REPORT_INTERVAL_MS = 250; const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000; let activeWebRtcControlChannel: RTCDataChannel | null = null; @@ -161,7 +160,6 @@ class WebRtcStreamClient implements StreamClientBackend { private controlChannel: RTCDataChannel | null = null; private diagnostics = createWebRtcDiagnostics(); private frameWatchdogTimeout = 0; - private lastStatsReportAt = 0; private lastVideoFrameAt = 0; private peerConnection: RTCPeerConnection | null = null; private reconnectTimeout = 0; @@ -194,7 +192,6 @@ class WebRtcStreamClient implements StreamClientBackend { const generation = ++this.connectGeneration; this.shouldReconnect = true; this.diagnostics = createWebRtcDiagnostics(); - this.lastStatsReportAt = 0; this.reportedVideoConfig = false; this.stats = createEmptyStreamStats(); this.onMessage({ @@ -616,22 +613,11 @@ class WebRtcStreamClient implements StreamClientBackend { this.stats.latestFrameGapMs = now - this.lastVideoFrameAt; } this.lastVideoFrameAt = now; - this.reportStats(now); + this.onMessage({ type: "stats", stats: { ...this.stats } }); } this.scheduleVideoFrame(); }; - private reportStats(now = performance.now()) { - if ( - this.lastStatsReportAt > 0 && - now - this.lastStatsReportAt < WEBRTC_STATS_REPORT_INTERVAL_MS - ) { - return; - } - this.lastStatsReportAt = now; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - } - private scheduleVideoFrame() { this.cancelVideoFrameCallback(); if (!this.video) { diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 0f306e4..b92af05 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -119,7 +119,7 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ id: "quality", label: "Quality", max_edge: 1440, - fps: 60, + fps: 30, min_bitrate: 3_000_000, bits_per_pixel: 4, }, diff --git a/server/src/main.rs b/server/src/main.rs index 0792207..db63894 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -627,7 +627,7 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "quality", max_edge: 1440, - fps: 60, + fps: 30, min_bitrate: 3_000_000, bits_per_pixel: 4, }), @@ -714,8 +714,8 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: true, - stream_quality_profile: Some("quality".to_owned()), + realtime_stream: false, + stream_quality_profile: None, } } } @@ -1199,7 +1199,6 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let port = choose_daemon_port_for_bind(4310, bind)?; let video_codec = VideoCodecMode::H264; let low_latency = false; - let stream_quality_profile = Some("quality".to_owned()); let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) .to_string(); @@ -1217,8 +1216,8 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: true, - stream_quality_profile: stream_quality_profile.clone(), + realtime_stream: false, + stream_quality_profile: None, }; write_daemon_metadata(&metadata)?; @@ -1240,7 +1239,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { None, video_codec, low_latency, - stream_quality_profile, + None, Some(access_token), Some(pairing_code), ); diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 8482b62..7ed04c6 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,13 +36,13 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(67); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(120); -const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(180); +const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); +const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); @@ -873,7 +873,7 @@ fn realtime_packet_pacing( packet_count: usize, realtime_stream: bool, ) -> Option<(usize, Duration)> { - if !realtime_stream || packet_count <= 1 { + if realtime_stream || packet_count <= 1 { return None; } let pacing_ticks = ((duration.as_millis() / 4).max(1) as usize).min(packet_count - 1); From 4c57870587cdee41c87d21253cdc3418b15321f2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:22:15 -0400 Subject: [PATCH 14/24] Uncap local hardware WebRTC streaming --- cli/XCWH264Encoder.m | 104 ++++++++++++++------------------- server/src/main.rs | 11 ++-- server/src/transport/webrtc.rs | 20 ++----- 3 files changed, 53 insertions(+), 82 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 3ee42f4..846218b 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -35,9 +35,6 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; -static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; -static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; - typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, @@ -601,7 +598,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; _codecType = XCWVideoCodecTypeForMode(_encoderMode); _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + _realtimeHardwareFrameIntervalUs = (_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); return self; } @@ -651,7 +648,7 @@ - (void)reconfigureForStreamQualityChange { self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; self->_softwareHealthyFrameCount = 0; - self->_realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + self->_realtimeHardwareFrameIntervalUs = (self->_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); self->_realtimeHardwarePacedFrameCount = 0; self->_realtimeHardwareHealthyFrameCount = 0; }); @@ -746,35 +743,42 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { return _lowLatencyMode ? XCWLowLatencySoftwareHealthyFrameWindow : XCWSoftwareHealthyFrameWindow; } +- (BOOL)fallbackToSoftwareEncoderIfHardwareUnavailableLocked { + if (_encoderMode != XCWVideoEncoderModeH264Hardware) { + return NO; + } + + _encoderMode = XCWVideoEncoderModeH264Software; + _lowLatencyMode = XCWLowLatencyModeFromEnvironment(); + _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; + _codecType = XCWVideoCodecTypeForMode(_encoderMode); + _hardwareAccelerated = NO; + _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; + _realtimeHardwareFrameIntervalUs = 0; + _lastSoftwareSubmissionUs = 0; + _lastRealtimeHardwareSubmissionUs = 0; + _softwareHealthyFrameCount = 0; + _realtimeHardwareHealthyFrameCount = 0; + _needsKeyFrame = YES; + return YES; +} + - (int32_t)expectedFrameRateLocked { + if (_encoderMode == XCWVideoEncoderModeH264Hardware) { + return XCWTargetRealTimeFrameRate; + } if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { return XCWTargetLowLatencySoftwareFrameRate; } return _realtimeStreamMode ? XCWRealtimeTargetFrameRate() : XCWTargetSoftwareFrameRate; } - if (_realtimeStreamMode) { - return XCWRealtimeTargetFrameRate(); - } return XCWTargetRealTimeFrameRate; } - (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs { - if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || _needsKeyFrame) { - return NO; - } - if (_realtimeHardwareFrameIntervalUs == 0) { - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - } - if (_lastRealtimeHardwareSubmissionUs == 0) { - return NO; - } - uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0; - if (elapsedUs >= _realtimeHardwareFrameIntervalUs) { - return NO; - } - _realtimeHardwarePacedFrameCount += 1; - return YES; + (void)nowUs; + return NO; } - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { @@ -847,39 +851,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { - return; - } - if (_realtimeHardwareFrameIntervalUs == 0) { - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - } - - uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); - uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); - if (latencyUs > _realtimeHardwareFrameIntervalUs) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; - uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; - if (nextIntervalUs < latencyBoundIntervalUs) { - nextIntervalUs = latencyBoundIntervalUs; - } - _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - return; - } - - if (latencyUs < _realtimeHardwareFrameIntervalUs && - _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { - _realtimeHardwareHealthyFrameCount += 1; - if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs - ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs - : minimumIntervalUs; - _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - } - return; - } - + (void)latencyUs; _realtimeHardwareHealthyFrameCount = 0; } @@ -1064,6 +1036,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height &session); _lastSessionStatus = status; if (status != noErr || session == NULL) { + if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { + return NO; + } return NO; } @@ -1118,10 +1093,12 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, kCFBooleanTrue); } - if (@available(macOS 15.0, *)) { - VTSessionSetProperty(session, - kVTCompressionPropertyKey_MaximumRealTimeFrameRate, - (__bridge CFTypeRef)@(expectedFrameRate)); + if (_encoderMode != XCWVideoEncoderModeH264Hardware) { + if (@available(macOS 15.0, *)) { + VTSessionSetProperty(session, + kVTCompressionPropertyKey_MaximumRealTimeFrameRate, + (__bridge CFTypeRef)@(expectedFrameRate)); + } } #ifdef kVTCompressionPropertyKey_MaxFrameDelayCount VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxFrameDelayCount, (__bridge CFTypeRef)@0); @@ -1131,9 +1108,14 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height _lastPrepareStatus = status; if (status != noErr) { [self invalidateCompressionSessionLocked]; + if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { + return NO; + } return NO; } - _hardwareAccelerated = XCWCompressionSessionUsesHardwareEncoder(session); + _hardwareAccelerated = (_encoderMode == XCWVideoEncoderModeH264Hardware) + ? YES + : XCWCompressionSessionUsesHardwareEncoder(session); return YES; } diff --git a/server/src/main.rs b/server/src/main.rs index db63894..b3803c7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -714,8 +714,8 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: false, - stream_quality_profile: None, + realtime_stream: true, + stream_quality_profile: Some("quality".to_owned()), } } } @@ -1199,6 +1199,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let port = choose_daemon_port_for_bind(4310, bind)?; let video_codec = VideoCodecMode::H264; let low_latency = false; + let stream_quality_profile = Some("quality".to_owned()); let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) .to_string(); @@ -1216,8 +1217,8 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: false, - stream_quality_profile: None, + realtime_stream: true, + stream_quality_profile: stream_quality_profile.clone(), }; write_daemon_metadata(&metadata)?; @@ -1239,7 +1240,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { None, video_codec, low_latency, - None, + stream_quality_profile, Some(access_token), Some(pairing_code), ); diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 7ed04c6..f247596 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -36,8 +36,8 @@ const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; -const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(67); -const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); +const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(100); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); @@ -873,7 +873,7 @@ fn realtime_packet_pacing( packet_count: usize, realtime_stream: bool, ) -> Option<(usize, Duration)> { - if realtime_stream || packet_count <= 1 { + if !realtime_stream || packet_count <= 1 { return None; } let pacing_ticks = ((duration.as_millis() / 4).max(1) as usize).min(packet_count - 1); @@ -1015,10 +1015,7 @@ impl WebRtcSendTiming { frame: &crate::transport::packet::FramePacket, realtime_stream: bool, ) -> Duration { - if realtime_stream { - self.last_timestamp_us = Some(frame.timestamp_us); - return realtime_sample_duration(); - } + let _ = realtime_stream; const MIN_FRAME_DURATION_US: u64 = 1_000; const DEFAULT_FRAME_DURATION_US: u64 = 16_667; @@ -1035,15 +1032,6 @@ impl WebRtcSendTiming { } } -fn realtime_sample_duration() -> Duration { - let fps = std::env::var("SIMDECK_REALTIME_FPS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(30) - .clamp(15, 60); - Duration::from_micros(1_000_000 / fps) -} - struct WebRtcMetricsGuard { metrics: Arc, } From 522bf2b922877e85b7fcf5a25f3805edd0a786d6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:29:12 -0400 Subject: [PATCH 15/24] Restore main WebRTC streaming implementation --- cli/XCWH264Encoder.m | 296 ++++-------------- .../src/features/stream/streamWorkerClient.ts | 163 +--------- client/src/features/stream/useLiveStream.ts | 4 +- server/src/transport/webrtc.rs | 18 +- 4 files changed, 90 insertions(+), 391 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 846218b..a2987b6 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -2,7 +2,6 @@ #import #import -#import #import #import #import @@ -35,10 +34,12 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; +static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; +static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; + typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, - XCWVideoEncoderModeJPEG, }; static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { @@ -50,9 +51,6 @@ static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { if ([value isEqualToString:@"h264-software"] || [value isEqualToString:@"software-h264"]) { return XCWVideoEncoderModeH264Software; } - if ([value isEqualToString:@"jpeg"] || [value isEqualToString:@"jpg"] || [value isEqualToString:@"mjpeg"]) { - return XCWVideoEncoderModeJPEG; - } return XCWVideoEncoderModeH264Software; } @@ -115,14 +113,14 @@ static int64_t XCWInt64FromEnvironment(NSString *name, int64_t fallback, int64_t static int32_t XCWRealtimeMaximumEncodedDimension(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_MAX_EDGE", XCWMaximumRealtimeHardwareEncodedDimension, - 320, + 720, XCWMaximumEncodedDimension); } static int32_t XCWRealtimeTargetFrameRate(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_FPS", XCWTargetRealtimeHardwareFrameRate, - 10, + 15, 60); } @@ -145,116 +143,14 @@ static int64_t XCWRealtimeBitsPerPixelBudgetValue(void) { static int32_t XCWRealtimeMinimumAverageBitRate(void) { return XCWIntFromEnvironment(@"SIMDECK_REALTIME_MIN_BITRATE", XCWMinimumRealtimeAverageBitRate, - 200000, + 750000, 20000000); } -static double XCWRealtimeKeyFrameIntervalSeconds(void) { - NSString *value = NSProcessInfo.processInfo.environment[@"SIMDECK_REALTIME_KEYFRAME_INTERVAL_SECONDS"]; - if (value.length == 0) { - return 4.0; - } - return fmin(10.0, fmax(1.0, value.doubleValue)); -} - -static CGFloat XCWJPEGQualityFromEnvironment(void) { - NSString *value = NSProcessInfo.processInfo.environment[@"SIMDECK_JPEG_QUALITY"]; - double quality = value.length > 0 ? value.doubleValue : 0.7; - if (!isfinite(quality)) { - return 0.7; - } - return (CGFloat)fmin(1.0, fmax(0.1, quality)); -} - -static CGColorSpaceRef XCWDeviceRGBColorSpace(void) { - static CGColorSpaceRef colorSpace = NULL; - static dispatch_once_t onceToken; - dispatch_once(&onceToken, ^{ - colorSpace = CGColorSpaceCreateDeviceRGB(); - }); - return colorSpace; -} - -static NSData *XCWJPEGDataFromPixelBuffer(CVPixelBufferRef pixelBuffer) { - if (pixelBuffer == NULL) { - return nil; - } - - CGImageRef image = NULL; - BOOL didLockPixelBuffer = NO; - OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - if (pixelFormat == kCVPixelFormatType_32BGRA && - CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) == kCVReturnSuccess) { - didLockPixelBuffer = YES; - void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); - size_t width = CVPixelBufferGetWidth(pixelBuffer); - size_t height = CVPixelBufferGetHeight(pixelBuffer); - size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); - if (baseAddress != NULL && width > 0 && height > 0 && bytesPerRow >= width * 4) { - CGDataProviderRef provider = CGDataProviderCreateWithData(NULL, - baseAddress, - bytesPerRow * height, - NULL); - if (provider != NULL) { - image = CGImageCreate(width, - height, - 8, - 32, - bytesPerRow, - XCWDeviceRGBColorSpace(), - kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, - provider, - NULL, - false, - kCGRenderingIntentDefault); - CGDataProviderRelease(provider); - } - } - } - - if (image == NULL) { - if (didLockPixelBuffer) { - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - didLockPixelBuffer = NO; - } - OSStatus imageStatus = VTCreateCGImageFromCVPixelBuffer(pixelBuffer, NULL, &image); - if (imageStatus != noErr || image == NULL) { - return nil; - } - } - - NSMutableData *data = [NSMutableData data]; - CGImageDestinationRef destination = - CGImageDestinationCreateWithData((__bridge CFMutableDataRef)data, - CFSTR("public.jpeg"), - 1, - NULL); - if (destination == NULL) { - CGImageRelease(image); - if (didLockPixelBuffer) { - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - } - return nil; - } - - NSDictionary *properties = @{ - (__bridge NSString *)kCGImageDestinationLossyCompressionQuality: @(XCWJPEGQualityFromEnvironment()), - }; - CGImageDestinationAddImage(destination, image, (__bridge CFDictionaryRef)properties); - BOOL ok = CGImageDestinationFinalize(destination); - CFRelease(destination); - CGImageRelease(image); - if (didLockPixelBuffer) { - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - } - return ok && data.length > 0 ? data : nil; -} - static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { switch (mode) { case XCWVideoEncoderModeH264Hardware: case XCWVideoEncoderModeH264Software: - case XCWVideoEncoderModeJPEG: default: return kCMVideoCodecType_H264; } @@ -265,9 +161,6 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { case XCWVideoEncoderModeH264Hardware: return @"h264"; case XCWVideoEncoderModeH264Software: - return @"h264-software"; - case XCWVideoEncoderModeJPEG: - return @"jpeg"; default: return @"h264-software"; } @@ -279,8 +172,6 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { return nil; case XCWVideoEncoderModeH264Software: return @"com.apple.videotoolbox.videoencoder.h264"; - case XCWVideoEncoderModeJPEG: - return nil; default: return nil; } @@ -598,7 +489,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; _codecType = XCWVideoCodecTypeForMode(_encoderMode); _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = (_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); return self; } @@ -648,7 +539,7 @@ - (void)reconfigureForStreamQualityChange { self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; self->_softwareHealthyFrameCount = 0; - self->_realtimeHardwareFrameIntervalUs = (self->_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); + self->_realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); self->_realtimeHardwarePacedFrameCount = 0; self->_realtimeHardwareHealthyFrameCount = 0; }); @@ -743,42 +634,35 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { return _lowLatencyMode ? XCWLowLatencySoftwareHealthyFrameWindow : XCWSoftwareHealthyFrameWindow; } -- (BOOL)fallbackToSoftwareEncoderIfHardwareUnavailableLocked { - if (_encoderMode != XCWVideoEncoderModeH264Hardware) { - return NO; - } - - _encoderMode = XCWVideoEncoderModeH264Software; - _lowLatencyMode = XCWLowLatencyModeFromEnvironment(); - _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; - _codecType = XCWVideoCodecTypeForMode(_encoderMode); - _hardwareAccelerated = NO; - _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = 0; - _lastSoftwareSubmissionUs = 0; - _lastRealtimeHardwareSubmissionUs = 0; - _softwareHealthyFrameCount = 0; - _realtimeHardwareHealthyFrameCount = 0; - _needsKeyFrame = YES; - return YES; -} - - (int32_t)expectedFrameRateLocked { - if (_encoderMode == XCWVideoEncoderModeH264Hardware) { - return XCWTargetRealTimeFrameRate; - } if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { return XCWTargetLowLatencySoftwareFrameRate; } return _realtimeStreamMode ? XCWRealtimeTargetFrameRate() : XCWTargetSoftwareFrameRate; } + if (_realtimeStreamMode) { + return XCWRealtimeTargetFrameRate(); + } return XCWTargetRealTimeFrameRate; } - (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs { - (void)nowUs; - return NO; + if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || _needsKeyFrame) { + return NO; + } + if (_realtimeHardwareFrameIntervalUs == 0) { + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + } + if (_lastRealtimeHardwareSubmissionUs == 0) { + return NO; + } + uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0; + if (elapsedUs >= _realtimeHardwareFrameIntervalUs) { + return NO; + } + _realtimeHardwarePacedFrameCount += 1; + return YES; } - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { @@ -799,18 +683,6 @@ - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { return YES; } -- (BOOL)shouldPaceJPEGFrameAtTimeUs:(uint64_t)nowUs { - if (_encoderMode != XCWVideoEncoderModeJPEG) { - return NO; - } - uint64_t intervalUs = XCWRealtimeFrameIntervalUs(); - if (_lastSoftwareSubmissionUs == 0) { - return NO; - } - uint64_t elapsedUs = nowUs >= _lastSoftwareSubmissionUs ? nowUs - _lastSoftwareSubmissionUs : 0; - return elapsedUs < intervalUs; -} - - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { if (_encoderMode != XCWVideoEncoderModeH264Software || latencyUs == 0) { return; @@ -851,7 +723,39 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { - (void)latencyUs; + if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { + return; + } + if (_realtimeHardwareFrameIntervalUs == 0) { + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + } + + uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); + uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); + if (latencyUs > _realtimeHardwareFrameIntervalUs) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; + uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; + if (nextIntervalUs < latencyBoundIntervalUs) { + nextIntervalUs = latencyBoundIntervalUs; + } + _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + return; + } + + if (latencyUs < _realtimeHardwareFrameIntervalUs && + _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { + _realtimeHardwareHealthyFrameCount += 1; + if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs + ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs + : minimumIntervalUs; + _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + } + return; + } + _realtimeHardwareHealthyFrameCount = 0; } @@ -893,26 +797,10 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { } uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); - if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || - [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs] || - [self shouldPaceJPEGFrameAtTimeUs:nowUs]) { + if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs]) { return YES; } - if (_encoderMode == XCWVideoEncoderModeJPEG) { - CVPixelBufferRef encodePixelBuffer = [self copyScaledPixelBufferIfNeeded:pixelBuffer - targetWidth:targetWidth - targetHeight:targetHeight]; - if (encodePixelBuffer == NULL) { - return NO; - } - BOOL ok = [self encodeJPEGPixelBufferLocked:encodePixelBuffer - sourceWidth:targetWidth - sourceHeight:targetHeight]; - CVPixelBufferRelease(encodePixelBuffer); - return ok; - } - if (![self ensureCompressionSessionWithWidth:targetWidth height:targetHeight]) { return NO; } @@ -965,41 +853,6 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { return YES; } -- (BOOL)encodeJPEGPixelBufferLocked:(CVPixelBufferRef)pixelBuffer - sourceWidth:(int32_t)sourceWidth - sourceHeight:(int32_t)sourceHeight { - uint64_t submittedAtUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); - if (_timestampOriginUs == 0) { - _timestampOriginUs = submittedAtUs; - } - uint64_t relativeTimestampUs = submittedAtUs - _timestampOriginUs; - - NSData *jpegData = XCWJPEGDataFromPixelBuffer(pixelBuffer); - if (jpegData.length == 0) { - _encodeFailureCount += 1; - _lastEncodeStatus = -1; - return NO; - } - - _width = sourceWidth; - _height = sourceHeight; - _lastSoftwareSubmissionUs = submittedAtUs; - _submittedFrameCount += 1; - _outputFrameCount += 1; - _keyFrameOutputCount += 1; - _lastEncodeStatus = noErr; - uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); - _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; - - self.outputHandler(jpegData, - relativeTimestampUs, - YES, - @"jpeg", - nil, - CGSizeMake(sourceWidth, sourceHeight)); - return YES; -} - - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height { if (_compressionSession != NULL && _width == width && _height == height) { return YES; @@ -1036,9 +889,6 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height &session); _lastSessionStatus = status; if (status != noErr || session == NULL) { - if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { - return NO; - } return NO; } @@ -1074,12 +924,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height VTSessionSetProperty(session, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel); } VTSessionSetProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(expectedFrameRate)); - double keyframeIntervalSeconds = _realtimeStreamMode - ? XCWRealtimeKeyFrameIntervalSeconds() - : (_lowLatencyMode ? 1.0 : 2.0); - int keyframeIntervalFrames = MAX(1, (int)llround((double)expectedFrameRate * keyframeIntervalSeconds)); - VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(keyframeIntervalFrames)); - VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(keyframeIntervalSeconds)); + BOOL shortKeyframeInterval = _lowLatencyMode || _realtimeStreamMode; + VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(shortKeyframeInterval ? expectedFrameRate : expectedFrameRate * 2)); + VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(shortKeyframeInterval ? 1.0 : 2.0)); VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(averageBitRate)); if (_realtimeStreamMode) { NSArray *dataRateLimits = @[ @@ -1093,12 +940,10 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, kCFBooleanTrue); } - if (_encoderMode != XCWVideoEncoderModeH264Hardware) { - if (@available(macOS 15.0, *)) { - VTSessionSetProperty(session, - kVTCompressionPropertyKey_MaximumRealTimeFrameRate, - (__bridge CFTypeRef)@(expectedFrameRate)); - } + if (@available(macOS 15.0, *)) { + VTSessionSetProperty(session, + kVTCompressionPropertyKey_MaximumRealTimeFrameRate, + (__bridge CFTypeRef)@(expectedFrameRate)); } #ifdef kVTCompressionPropertyKey_MaxFrameDelayCount VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxFrameDelayCount, (__bridge CFTypeRef)@0); @@ -1108,14 +953,9 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height _lastPrepareStatus = status; if (status != noErr) { [self invalidateCompressionSessionLocked]; - if ([self fallbackToSoftwareEncoderIfHardwareUnavailableLocked]) { - return NO; - } return NO; } - _hardwareAccelerated = (_encoderMode == XCWVideoEncoderModeH264Hardware) - ? YES - : XCWCompressionSessionUsesHardwareEncoder(session); + _hardwareAccelerated = XCWCompressionSessionUsesHardwareEncoder(session); return YES; } diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 1acb980..6cf4906 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -15,7 +15,7 @@ const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000; let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeStreamClient: StreamWorkerClient | null = null; -export type StreamBackend = "mjpeg-img" | "webrtc"; +export type StreamBackend = "webrtc"; export function sendWebRtcControlMessage(encoded: string): boolean { if (activeWebRtcControlChannel?.readyState !== "open") { @@ -41,118 +41,6 @@ interface StreamClientBackend { disconnect(): void; } -class MjpegImageStreamClient implements StreamClientBackend { - private canvas: HTMLCanvasElement | null = null; - private connectGeneration = 0; - private image: HTMLImageElement | null = null; - private statsInterval = 0; - private stats: StreamStats = createEmptyStreamStats(); - - constructor( - private readonly onMessage: (message: WorkerToMainMessage) => void, - ) {} - - attachCanvas(canvasElement: HTMLCanvasElement) { - this.canvas = canvasElement; - } - - clear() { - this.canvas - ?.getContext("2d") - ?.clearRect(0, 0, this.canvas.width, this.canvas.height); - } - - connect(target: StreamConnectTarget) { - this.disconnect(); - if (!this.canvas) { - return; - } - const canvasElement = this.canvas; - const generation = ++this.connectGeneration; - this.stats = { - ...createEmptyStreamStats(), - codec: "mjpeg", - }; - this.onMessage({ - type: "status", - status: { detail: "Opening MJPEG image stream", state: "connecting" }, - }); - - const image = document.createElement("img"); - image.alt = ""; - image.className = "stream-image"; - image.decoding = "async"; - image.draggable = false; - image.src = buildMjpegStreamUrl(target.udid); - image.addEventListener("load", () => { - if (generation !== this.connectGeneration) { - return; - } - this.reportImageSize(image); - this.stats.decodedFrames = Math.max(1, this.stats.decodedFrames); - this.stats.renderedFrames = Math.max(1, this.stats.renderedFrames); - this.stats.receivedPackets = Math.max(1, this.stats.receivedPackets); - this.onMessage({ type: "stats", stats: { ...this.stats } }); - this.onMessage({ type: "status", status: { state: "streaming" } }); - }); - image.addEventListener("error", () => { - if (generation !== this.connectGeneration) { - return; - } - this.onMessage({ - type: "status", - status: { error: "MJPEG image stream failed.", state: "error" }, - }); - }); - canvasElement.after(image); - this.image = image; - this.statsInterval = window.setInterval(() => { - if (generation !== this.connectGeneration || !this.image) { - return; - } - this.reportImageSize(this.image); - this.onMessage({ type: "stats", stats: { ...this.stats } }); - }, 1000); - } - - destroy() { - this.disconnect(); - } - - disconnect() { - this.connectGeneration += 1; - if (this.statsInterval) { - window.clearInterval(this.statsInterval); - this.statsInterval = 0; - } - if (this.image) { - this.image.removeAttribute("src"); - this.image.remove(); - this.image = null; - } - this.onMessage({ type: "status", status: { state: "idle" } }); - } - - private reportImageSize(image: HTMLImageElement) { - const width = image.naturalWidth; - const height = image.naturalHeight; - if (width <= 0 || height <= 0) { - return; - } - if (this.canvas) { - if (this.canvas.width !== width) { - this.canvas.width = width; - } - if (this.canvas.height !== height) { - this.canvas.height = height; - } - } - this.stats.width = width; - this.stats.height = height; - this.onMessage({ type: "video-config", size: { width, height } }); - } -} - class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; @@ -218,8 +106,7 @@ class WebRtcStreamClient implements StreamClientBackend { const controlChannel = peerConnection.createDataChannel( WEBRTC_CONTROL_CHANNEL_LABEL, { - maxPacketLifeTime: 250, - ordered: false, + ordered: true, }, ); this.controlChannel = controlChannel; @@ -381,10 +268,6 @@ class WebRtcStreamClient implements StreamClientBackend { type: "status", status: { error: message, state: "error" }, }); - if (isPermanentStreamFailure(message)) { - this.shouldReconnect = false; - return; - } this.scheduleReconnect(target, generation); } @@ -857,30 +740,6 @@ function candidateStatsSummary(candidate: RTCStats | undefined): string { return `${stats.candidateType ?? "?"}/${stats.protocol ?? "?"}/${stats.address || stats.ip ? "addr" : "noaddr"}/${stats.port ?? "?"}`; } -function buildMjpegStreamUrl(udid: string): string { - const url = new URL( - `/api/simulators/${encodeURIComponent(udid)}/mjpeg`, - window.location.href, - ); - const token = new URL(window.location.href).searchParams.get("simdeckToken"); - if (token) { - url.searchParams.set("simdeckToken", token); - } - url.searchParams.set("cacheBust", String(Date.now())); - return url.toString(); -} - -function isPermanentStreamFailure(message: string): boolean { - const normalized = message.toLowerCase(); - return ( - normalized.includes("private simulator display attach previously failed") || - normalized.includes("coresimulator did not provide") || - normalized.includes("did not expose any live screens") || - normalized.includes("headless screen") || - normalized.includes("screen adapter") - ); -} - function waitForIceGathering(peerConnection: RTCPeerConnection) { if (peerConnection.iceGatheringState === "complete") { return Promise.resolve(); @@ -899,7 +758,6 @@ function waitForIceGathering(peerConnection: RTCPeerConnection) { export class StreamWorkerClient { private readonly onMessage: (message: WorkerToMainMessage) => void; private backend: StreamClientBackend | null = null; - private canvas: HTMLCanvasElement | null = null; private attachedCanvas = false; private disposed = false; @@ -916,24 +774,13 @@ export class StreamWorkerClient { return; } - this.canvas = canvasElement; + this.backend = new WebRtcStreamClient(this.onMessage); + this.backend.attachCanvas(canvasElement); this.attachedCanvas = true; } - async connect(target: StreamConnectTarget) { + connect(target: StreamConnectTarget) { try { - const health = await fetchHealth().catch(() => null); - const shouldUseMjpeg = health?.videoCodec === "jpeg"; - const isMjpegBackend = this.backend instanceof MjpegImageStreamClient; - if (!this.backend || shouldUseMjpeg !== isMjpegBackend) { - this.backend?.destroy(); - this.backend = shouldUseMjpeg - ? new MjpegImageStreamClient(this.onMessage) - : new WebRtcStreamClient(this.onMessage); - if (this.canvas) { - this.backend.attachCanvas(this.canvas); - } - } const result = this.backend?.connect(target); if (result && typeof result.catch === "function") { result.catch((error: unknown) => { diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index c9ffdab..a07d923 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -288,7 +288,7 @@ export function useLiveStream({ runtimeInfo, stats, status, - streamBackend: stats.codec === "mjpeg" ? "mjpeg-img" : "webrtc", - streamCanvasKey: `stream-${streamCanvasRevision}`, + streamBackend: "webrtc", + streamCanvasKey: `webrtc-${streamCanvasRevision}`, }; } diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index f247596..9ed96e7 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -39,7 +39,7 @@ const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3; const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(100); const WEBRTC_LOW_LATENCY_REFRESH_INTERVAL: Duration = Duration::from_millis(67); -const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(134); +const WEBRTC_LOW_LATENCY_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); @@ -852,7 +852,7 @@ async fn write_frame_sample_with_timeout( if frame.is_keyframe { WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT } else { - WEBRTC_REALTIME_WRITE_TIMEOUT + WEBRTC_REALTIME_WRITE_TIMEOUT.max(realtime_sample_duration() * 2) } } else { WEBRTC_WRITE_TIMEOUT @@ -1015,7 +1015,10 @@ impl WebRtcSendTiming { frame: &crate::transport::packet::FramePacket, realtime_stream: bool, ) -> Duration { - let _ = realtime_stream; + if realtime_stream { + self.last_timestamp_us = Some(frame.timestamp_us); + return realtime_sample_duration(); + } const MIN_FRAME_DURATION_US: u64 = 1_000; const DEFAULT_FRAME_DURATION_US: u64 = 16_667; @@ -1032,6 +1035,15 @@ impl WebRtcSendTiming { } } +fn realtime_sample_duration() -> Duration { + let fps = std::env::var("SIMDECK_REALTIME_FPS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(30) + .clamp(15, 60); + Duration::from_micros(1_000_000 / fps) +} + struct WebRtcMetricsGuard { metrics: Arc, } From 0f3693062e7295e786958ad1b8b75da3f5ee6119 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:32:00 -0400 Subject: [PATCH 16/24] Remove local hardware stream FPS cap --- server/src/main.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index b3803c7..db63894 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -714,8 +714,8 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: true, - stream_quality_profile: Some("quality".to_owned()), + realtime_stream: false, + stream_quality_profile: None, } } } @@ -1199,7 +1199,6 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { let port = choose_daemon_port_for_bind(4310, bind)?; let video_codec = VideoCodecMode::H264; let low_latency = false; - let stream_quality_profile = Some("quality".to_owned()); let advertise_host = detect_lan_ip() .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) .to_string(); @@ -1217,8 +1216,8 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: true, - stream_quality_profile: stream_quality_profile.clone(), + realtime_stream: false, + stream_quality_profile: None, }; write_daemon_metadata(&metadata)?; @@ -1240,7 +1239,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { None, video_codec, low_latency, - stream_quality_profile, + None, Some(access_token), Some(pairing_code), ); From 95f8abf5c1665d501562ebf02083ee25c6fc9ce0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:34:29 -0400 Subject: [PATCH 17/24] Avoid reconnecting established WebRTC streams on frame stalls --- client/src/features/stream/streamWorkerClient.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 6cf4906..83f4814 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -304,7 +304,7 @@ class WebRtcStreamClient implements StreamClientBackend { const hasRenderedFrame = this.stats.renderedFrames > 0; const frameAgeMs = this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity; - if (!hasRenderedFrame || frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { + if (!hasRenderedFrame) { this.handleConnectionError( target, generation, @@ -312,6 +312,15 @@ class WebRtcStreamClient implements StreamClientBackend { ); return; } + if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { + this.onMessage({ + type: "status", + status: { + detail: "Waiting for fresh WebRTC frames", + state: "streaming", + }, + }); + } this.scheduleFrameWatchdog(target, generation); }, this.stats.renderedFrames > 0 From 6903fd826b53d5e0464e7e6270bae81a90e8e791 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:35:59 -0400 Subject: [PATCH 18/24] Tolerate transient WebRTC disconnected state --- client/src/features/stream/streamWorkerClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 83f4814..789b091 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -170,7 +170,7 @@ class WebRtcStreamClient implements StreamClientBackend { if ( generation === this.connectGeneration && (peerConnection.connectionState === "failed" || - peerConnection.connectionState === "disconnected") + peerConnection.connectionState === "closed") ) { if (peerConnection.connectionState === "failed") { void this.updateSelectedCandidatePair(peerConnection, target); From cc6e0ea5ad77f5989409c5d6f997abd186338c8e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:41:43 -0400 Subject: [PATCH 19/24] Use realtime cleanup without capping local hardware --- cli/XCWH264Encoder.m | 64 +++++----------------------------- server/src/main.rs | 5 +-- server/src/transport/webrtc.rs | 18 +++++++--- 3 files changed, 24 insertions(+), 63 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index a2987b6..15ca43a 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -34,9 +34,6 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; -static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; -static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; - typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, @@ -489,7 +486,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; _codecType = XCWVideoCodecTypeForMode(_encoderMode); _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + _realtimeHardwareFrameIntervalUs = (_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); return self; } @@ -539,7 +536,7 @@ - (void)reconfigureForStreamQualityChange { self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; self->_softwareHealthyFrameCount = 0; - self->_realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + self->_realtimeHardwareFrameIntervalUs = (self->_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); self->_realtimeHardwarePacedFrameCount = 0; self->_realtimeHardwareHealthyFrameCount = 0; }); @@ -635,34 +632,21 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { } - (int32_t)expectedFrameRateLocked { + if (_encoderMode == XCWVideoEncoderModeH264Hardware) { + return XCWTargetRealTimeFrameRate; + } if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { return XCWTargetLowLatencySoftwareFrameRate; } return _realtimeStreamMode ? XCWRealtimeTargetFrameRate() : XCWTargetSoftwareFrameRate; } - if (_realtimeStreamMode) { - return XCWRealtimeTargetFrameRate(); - } return XCWTargetRealTimeFrameRate; } - (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs { - if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || _needsKeyFrame) { - return NO; - } - if (_realtimeHardwareFrameIntervalUs == 0) { - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - } - if (_lastRealtimeHardwareSubmissionUs == 0) { - return NO; - } - uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0; - if (elapsedUs >= _realtimeHardwareFrameIntervalUs) { - return NO; - } - _realtimeHardwarePacedFrameCount += 1; - return YES; + (void)nowUs; + return NO; } - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { @@ -723,39 +707,7 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { - return; - } - if (_realtimeHardwareFrameIntervalUs == 0) { - _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); - } - - uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); - uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); - if (latencyUs > _realtimeHardwareFrameIntervalUs) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; - uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; - if (nextIntervalUs < latencyBoundIntervalUs) { - nextIntervalUs = latencyBoundIntervalUs; - } - _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - return; - } - - if (latencyUs < _realtimeHardwareFrameIntervalUs && - _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { - _realtimeHardwareHealthyFrameCount += 1; - if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { - uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs - ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs - : minimumIntervalUs; - _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); - _realtimeHardwareHealthyFrameCount = 0; - } - return; - } - + (void)latencyUs; _realtimeHardwareHealthyFrameCount = 0; } diff --git a/server/src/main.rs b/server/src/main.rs index db63894..d6a163e 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -714,7 +714,7 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: false, + realtime_stream: true, stream_quality_profile: None, } } @@ -1216,7 +1216,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: false, + realtime_stream: true, stream_quality_profile: None, }; write_daemon_metadata(&metadata)?; @@ -1232,6 +1232,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { println!("q or ^C to stop server"); let _ = io::stdout().flush(); + env::set_var("SIMDECK_REALTIME_STREAM", "1"); let result = serve_with_appkit( port, bind, diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 9ed96e7..68c37ee 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -44,9 +44,10 @@ const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; +const WEBRTC_DISCONNECTED_GRACE: Duration = Duration::from_secs(6); static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); -const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 3; +const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 8; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -597,6 +598,7 @@ impl WebRtcMediaStream { let mut refresh_sleep = Box::pin(time::sleep(refresh_floor)); let mut adaptive_refresh_interval = refresh_floor; let mut bootstrap_frames_remaining = WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS; + let mut disconnected_since: Option = None; let mut waiting_for_keyframe = false; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); @@ -642,6 +644,15 @@ impl WebRtcMediaStream { warn!("WebRTC media stream closing for {udid}: peer state {peer_state}"); break; } + if matches!(peer_state, RTCPeerConnectionState::Disconnected) { + let since = disconnected_since.get_or_insert_with(time::Instant::now); + if since.elapsed() >= WEBRTC_DISCONNECTED_GRACE { + warn!("WebRTC media stream closing for {udid}: peer disconnected for {:?}", since.elapsed()); + break; + } + } else { + disconnected_since = None; + } } _ = &mut bootstrap_sleep, if bootstrap_frames_remaining > 0 => { match write_frame_sample_with_timeout( @@ -1015,10 +1026,7 @@ impl WebRtcSendTiming { frame: &crate::transport::packet::FramePacket, realtime_stream: bool, ) -> Duration { - if realtime_stream { - self.last_timestamp_us = Some(frame.timestamp_us); - return realtime_sample_duration(); - } + let _ = realtime_stream; const MIN_FRAME_DURATION_US: u64 = 1_000; const DEFAULT_FRAME_DURATION_US: u64 = 16_667; From 65ec204bebd487fff1a20f6a2cad3a4c52ce1459 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 14:53:46 -0400 Subject: [PATCH 20/24] Restore smooth local WebRTC defaults --- cli/XCWH264Encoder.m | 63 ++++++++++++++++--- .../src/features/stream/streamWorkerClient.ts | 15 +++-- client/src/features/stream/useLiveStream.ts | 25 ++++++-- server/src/main.rs | 5 +- server/src/transport/webrtc.rs | 37 +++++++++-- 5 files changed, 118 insertions(+), 27 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 15ca43a..19f04a1 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -34,6 +34,8 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; +static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; +static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, @@ -486,7 +488,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; _codecType = XCWVideoCodecTypeForMode(_encoderMode); _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _realtimeHardwareFrameIntervalUs = (_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); return self; } @@ -536,7 +538,7 @@ - (void)reconfigureForStreamQualityChange { self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; self->_softwareHealthyFrameCount = 0; - self->_realtimeHardwareFrameIntervalUs = (self->_encoderMode == XCWVideoEncoderModeH264Hardware) ? 0 : XCWRealtimeFrameIntervalUs(); + self->_realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); self->_realtimeHardwarePacedFrameCount = 0; self->_realtimeHardwareHealthyFrameCount = 0; }); @@ -632,21 +634,34 @@ - (NSUInteger)softwareHealthyFrameWindowLocked { } - (int32_t)expectedFrameRateLocked { - if (_encoderMode == XCWVideoEncoderModeH264Hardware) { - return XCWTargetRealTimeFrameRate; - } if (_encoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { return XCWTargetLowLatencySoftwareFrameRate; } return _realtimeStreamMode ? XCWRealtimeTargetFrameRate() : XCWTargetSoftwareFrameRate; } + if (_realtimeStreamMode) { + return XCWRealtimeTargetFrameRate(); + } return XCWTargetRealTimeFrameRate; } - (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs { - (void)nowUs; - return NO; + if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || _needsKeyFrame) { + return NO; + } + if (_realtimeHardwareFrameIntervalUs == 0) { + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + } + if (_lastRealtimeHardwareSubmissionUs == 0) { + return NO; + } + uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0; + if (elapsedUs >= _realtimeHardwareFrameIntervalUs) { + return NO; + } + _realtimeHardwarePacedFrameCount += 1; + return YES; } - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { @@ -707,7 +722,39 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { } - (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs { - (void)latencyUs; + if (_encoderMode != XCWVideoEncoderModeH264Hardware || !_realtimeStreamMode || latencyUs == 0) { + return; + } + if (_realtimeHardwareFrameIntervalUs == 0) { + _realtimeHardwareFrameIntervalUs = XCWRealtimeFrameIntervalUs(); + } + + uint64_t minimumIntervalUs = XCWRealtimeFrameIntervalUs(); + uint64_t maximumIntervalUs = XCWRealtimeMaximumFrameIntervalUs(); + if (latencyUs > _realtimeHardwareFrameIntervalUs) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; + uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; + if (nextIntervalUs < latencyBoundIntervalUs) { + nextIntervalUs = latencyBoundIntervalUs; + } + _realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + return; + } + + if (latencyUs < _realtimeHardwareFrameIntervalUs && + _realtimeHardwareFrameIntervalUs > minimumIntervalUs) { + _realtimeHardwareHealthyFrameCount += 1; + if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { + uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs + ? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs + : minimumIntervalUs; + _realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); + _realtimeHardwareHealthyFrameCount = 0; + } + return; + } + _realtimeHardwareHealthyFrameCount = 0; } diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 789b091..cb16c61 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -170,7 +170,7 @@ class WebRtcStreamClient implements StreamClientBackend { if ( generation === this.connectGeneration && (peerConnection.connectionState === "failed" || - peerConnection.connectionState === "closed") + peerConnection.connectionState === "disconnected") ) { if (peerConnection.connectionState === "failed") { void this.updateSelectedCandidatePair(peerConnection, target); @@ -313,13 +313,12 @@ class WebRtcStreamClient implements StreamClientBackend { return; } if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { - this.onMessage({ - type: "status", - status: { - detail: "Waiting for fresh WebRTC frames", - state: "streaming", - }, - }); + this.handleConnectionError( + target, + generation, + new Error("WebRTC video stalled before rendering fresh frames."), + ); + return; } this.scheduleFrameWatchdog(target, generation); }, diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index a07d923..fd77985 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -77,13 +77,30 @@ export function useLiveStream({ const [status, setStatus] = useState({ state: "idle" }); const [error, setError] = useState(""); const [fps, setFps] = useState(0); + const [pageVisible, setPageVisible] = useState( + () => document.visibilityState !== "hidden", + ); const [streamCanvasRevision, setStreamCanvasRevision] = useState(0); const [runtimeInfo] = useState(detectRuntimeInfo); + const streamPaused = paused || !pageVisible; if (!clientTelemetryIdRef.current) { clientTelemetryIdRef.current = createClientTelemetryId(); } + useEffect(() => { + const updatePageVisible = () => { + setPageVisible(document.visibilityState !== "hidden"); + }; + document.addEventListener("visibilitychange", updatePageVisible); + window.addEventListener("pageshow", updatePageVisible); + updatePageVisible(); + return () => { + document.removeEventListener("visibilitychange", updatePageVisible); + window.removeEventListener("pageshow", updatePageVisible); + }; + }, []); + useEffect(() => { let frameCount = 0; let lastSampleAt = performance.now(); @@ -109,7 +126,7 @@ export function useLiveStream({ }, []); useEffect(() => { - if (paused || !canvasElement || workerClientRef.current) { + if (streamPaused || !canvasElement || workerClientRef.current) { return; } @@ -164,7 +181,7 @@ export function useLiveStream({ workerClient.destroy(); workerClientRef.current = null; }; - }, [canvasElement, paused]); + }, [canvasElement, streamPaused]); useEffect(() => { latestDecodedFramesRef.current = stats.decodedFrames; @@ -219,7 +236,7 @@ export function useLiveStream({ setError(""); setFps(0); - if (paused || !simulator?.isBooted) { + if (streamPaused || !simulator?.isBooted) { workerClient.disconnect(); workerClient.clear(); return; @@ -237,7 +254,7 @@ export function useLiveStream({ return () => { workerClient.disconnect(); }; - }, [canvasElement, simulator?.isBooted, simulator?.udid, paused]); + }, [canvasElement, simulator?.isBooted, simulator?.udid, streamPaused]); useEffect(() => { if (!simulator?.udid) { diff --git a/server/src/main.rs b/server/src/main.rs index d6a163e..db63894 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -714,7 +714,7 @@ impl Default for DaemonLaunchOptions { client_root: None, video_codec: VideoCodecMode::H264, low_latency: false, - realtime_stream: true, + realtime_stream: false, stream_quality_profile: None, } } @@ -1216,7 +1216,7 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { log_path: None, video_codec: Some(video_codec.as_env_value().to_owned()), low_latency, - realtime_stream: true, + realtime_stream: false, stream_quality_profile: None, }; write_daemon_metadata(&metadata)?; @@ -1232,7 +1232,6 @@ fn run_foreground_ui(selector: Option) -> anyhow::Result<()> { println!("q or ^C to stop server"); let _ = io::stdout().flush(); - env::set_var("SIMDECK_REALTIME_STREAM", "1"); let result = serve_with_appkit( port, bind, diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 68c37ee..8e6894e 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -47,7 +47,7 @@ const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; const WEBRTC_DISCONNECTED_GRACE: Duration = Duration::from_secs(6); static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); -const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 8; +const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 3; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -160,7 +160,13 @@ pub async fn create_answer( .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, ); register_diagnostics(&peer_connection, &udid); - register_control_data_channel(&peer_connection, session.clone(), udid.clone()); + let (viewer_closed_token, viewer_closed) = broadcast::channel(1); + register_control_data_channel( + &peer_connection, + session.clone(), + udid.clone(), + viewer_closed_token.clone(), + ); let video_track = Arc::new(TrackLocalStaticRTP::new( RTCRtpCodecCapability { @@ -221,6 +227,7 @@ pub async fn create_answer( video_track, cancellation_token, cancellation, + viewer_closed, } .run(), ); @@ -332,15 +339,17 @@ fn register_control_data_channel( peer_connection: &Arc, session: crate::simulators::session::SimulatorSession, udid: String, + viewer_closed_token: broadcast::Sender<()>, ) { peer_connection.on_data_channel(Box::new(move |channel: Arc| { let session = session.clone(); let udid = udid.clone(); + let viewer_closed_token = viewer_closed_token.clone(); Box::pin(async move { if channel.label() != WEBRTC_CONTROL_CHANNEL_LABEL { return; } - attach_control_data_channel(channel, session, udid); + attach_control_data_channel(channel, session, udid, viewer_closed_token); }) })); } @@ -349,7 +358,18 @@ fn attach_control_data_channel( channel: Arc, session: crate::simulators::session::SimulatorSession, udid: String, + viewer_closed_token: broadcast::Sender<()>, ) { + let close_udid = udid.clone(); + channel.on_close(Box::new(move || { + let close_udid = close_udid.clone(); + let viewer_closed_token = viewer_closed_token.clone(); + Box::pin(async move { + info!("WebRTC control data channel closed for {close_udid}"); + let _ = viewer_closed_token.send(()); + }) + })); + channel.on_message(Box::new(move |message: DataChannelMessage| { let session = session.clone(); let udid = udid.clone(); @@ -565,6 +585,7 @@ struct WebRtcMediaStream { video_track: Arc, cancellation_token: broadcast::Sender<()>, cancellation: broadcast::Receiver<()>, + viewer_closed: broadcast::Receiver<()>, } impl WebRtcMediaStream { @@ -578,6 +599,7 @@ impl WebRtcMediaStream { video_track, cancellation_token, mut cancellation, + mut viewer_closed, } = self; let mut rx = session.subscribe(); let mut latest_keyframe = first_frame.clone(); @@ -638,6 +660,10 @@ impl WebRtcMediaStream { warn!("WebRTC media stream replaced for {udid}"); break; } + _ = viewer_closed.recv() => { + info!("WebRTC media stream closing for {udid}: control channel closed"); + break; + } _ = peer_state_interval.tick() => { let peer_state = peer_connection.connection_state(); if matches!(peer_state, RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) { @@ -1026,7 +1052,10 @@ impl WebRtcSendTiming { frame: &crate::transport::packet::FramePacket, realtime_stream: bool, ) -> Duration { - let _ = realtime_stream; + if realtime_stream { + self.last_timestamp_us = Some(frame.timestamp_us); + return realtime_sample_duration(); + } const MIN_FRAME_DURATION_US: u64 = 1_000; const DEFAULT_FRAME_DURATION_US: u64 = 16_667; From 5d20181b2db1f0e54c1a34ea491d1e6b52b03c4e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 15:08:28 -0400 Subject: [PATCH 21/24] Restore main local preview hot path --- cli/DFPrivateSimulatorDisplayBridge.m | 105 +++++------------- cli/XCWH264Encoder.m | 1 + .../src/features/stream/streamWorkerClient.ts | 10 +- client/src/features/stream/useLiveStream.ts | 25 +---- server/src/simulators/session.rs | 6 - server/src/transport/webrtc.rs | 41 +------ 6 files changed, 36 insertions(+), 152 deletions(-) diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 683c355..7794843 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -617,35 +617,16 @@ static id DFInitSimDeviceScreen(Class screenClass, id device, uint32_t screenID, return @{}; } - SEL screensSelector = sel_registerName("screens"); - id screens = nil; - if ([adapter respondsToSelector:screensSelector]) { - @try { - screens = ((id(*)(id, SEL))objc_msgSend)(adapter, screensSelector); - } @catch (NSException *exception) { - DFLog(@"SimulatorKit screen adapter screens selector threw: %@", exception.reason ?: exception.name); - screens = nil; - } - } - if (screens == nil) { - Ivar screensIvar = class_getInstanceVariable([adapter class], "_screens"); - if (screensIvar != NULL) { - screens = object_getIvar(adapter, screensIvar); - } - } - // The full mangled tail of `SimDeviceScreenAdapter.screens.getter` drifts // across Xcode releases (Xcode 26.4 retyped it from // `[UInt32: SimScreen]` (ObjC) to `[UInt32: SimDeviceScreen]` (Swift)). // Resolve by stable prefix instead. If the values are now SimDeviceScreen // wrappers, unwrap each via `.screen` so callers keep talking to a // SimScreen-shaped object. - if (screens == nil) { - screens = DFCallSwiftSelfGetter( - adapter, - "$s12SimulatorKit22SimDeviceScreenAdapterC7screensSDys6UInt32VSo0cE0_pGvg" - ); - } + id screens = DFCallSwiftSelfGetter( + adapter, + "$s12SimulatorKit22SimDeviceScreenAdapterC7screensSDys6UInt32VSo0cE0_pGvg" + ); if (screens == nil) { screens = DFCallSwiftSelfGetter( adapter, @@ -2448,34 +2429,9 @@ - (nullable instancetype)initWithUDID:(NSString *)udid DFLog(@"Initialized bootstrap SimDeviceScreen for %@", udid); } [self updateStatus:@"Waiting for CoreSimulator screen adapter"]; - NSDate *adapterDeadline = [NSDate dateWithTimeIntervalSinceNow:15.0]; - while (_screenAdapter == nil && [adapterDeadline timeIntervalSinceNow] > 0) { - if ([NSStringFromClass([_screenAdapterHost class]) containsString:@"SimDeviceScreenAdapter"]) { - _screenAdapter = _screenAdapterHost; - } else { - Ivar screenAdapterIvar = class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter"); - if (screenAdapterIvar != NULL) { - _screenAdapter = object_getIvar(_screenAdapterHost, screenAdapterIvar); - } - } - if (_screenAdapter != nil) { - break; - } + DFSpinRunLoop(0.5); - // On cold CoreSimulator boots the Swift screenAdapter host can exist - // before its private ROCK proxy is connected. Re-read the extension - // property while spinning the run loop instead of failing the attach. - id refreshedHost = DFCallSwiftSelfGetterByPattern( - _device, - "$sSo9SimDeviceC12SimulatorKitE13screenAdapter", - "vg", - "SimDevice.screenAdapter.getter (SimulatorKit extension)" - ); - if (refreshedHost != nil) { - _screenAdapterHost = refreshedHost; - } - DFSpinRunLoop(0.2); - } + _screenAdapter = object_getIvar(_screenAdapterHost, class_getInstanceVariable([_screenAdapterHost class], "_screenAdapter")); if (_screenAdapter == nil) { DFLog(@"SimulatorKit screen adapter host did not expose _screenAdapter for %@; host class=%@", udid, NSStringFromClass([_screenAdapterHost class])); if (error != NULL) { @@ -2489,33 +2445,28 @@ - (nullable instancetype)initWithUDID:(NSString *)udid _screenAdapterCallbackUUID = [NSUUID UUID]; __weak typeof(self) weakSelf = self; - SEL registerCallbacksSelector = sel_registerName("registerScreenAdapterCallbacksWithUUID:callbackQueue:screenConnectedCallback:screenWillDisconnectCallback:"); - if ([_screenAdapter respondsToSelector:registerCallbacksSelector]) { - ((void(*)(id, SEL, id, id, id, id))objc_msgSend)( - _screenAdapter, - registerCallbacksSelector, - _screenAdapterCallbackUUID, - _callbackQueue, - ^(id simScreen) { - (void)simScreen; - __strong typeof(weakSelf) strongSelf = weakSelf; - if (strongSelf == nil) { - return; - } - [strongSelf updateStatus:@"CoreSimulator screen proxy connected"]; - }, - ^(id simScreen) { - (void)simScreen; - __strong typeof(weakSelf) strongSelf = weakSelf; - if (strongSelf == nil) { - return; - } - [strongSelf updateStatus:@"CoreSimulator screen proxy disconnected"]; + ((void(*)(id, SEL, id, id, id, id))objc_msgSend)( + _screenAdapter, + sel_registerName("registerScreenAdapterCallbacksWithUUID:callbackQueue:screenConnectedCallback:screenWillDisconnectCallback:"), + _screenAdapterCallbackUUID, + _callbackQueue, + ^(id simScreen) { + (void)simScreen; + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil) { + return; } - ); - } else { - DFLog(@"SimulatorKit screen adapter %@ does not expose callback registration; polling screens directly", NSStringFromClass([_screenAdapter class])); - } + [strongSelf updateStatus:@"CoreSimulator screen proxy connected"]; + }, + ^(id simScreen) { + (void)simScreen; + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil) { + return; + } + [strongSelf updateStatus:@"CoreSimulator screen proxy disconnected"]; + } + ); [self updateStatus:@"Waiting for headless simulator screens"]; @@ -2524,7 +2475,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid // seconds (or only after Simulator.app primes the device). Poll instead of // relying on a fixed 0.5s sleep. NSDictionary *screens = DFReadAvailableAdapterScreens(_screenAdapterHost, _screenAdapter); - NSDate *screenDeadline = [NSDate dateWithTimeIntervalSinceNow:20.0]; + NSDate *screenDeadline = [NSDate dateWithTimeIntervalSinceNow:10.0]; while (screens.count == 0 && [screenDeadline timeIntervalSinceNow] > 0) { DFSpinRunLoop(0.1); screens = DFReadAvailableAdapterScreens(_screenAdapterHost, _screenAdapter); diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 19f04a1..a2987b6 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -36,6 +36,7 @@ static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; + typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index cb16c61..6cf4906 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -304,15 +304,7 @@ class WebRtcStreamClient implements StreamClientBackend { const hasRenderedFrame = this.stats.renderedFrames > 0; const frameAgeMs = this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity; - if (!hasRenderedFrame) { - this.handleConnectionError( - target, - generation, - new Error("WebRTC video stalled before rendering fresh frames."), - ); - return; - } - if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { + if (!hasRenderedFrame || frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { this.handleConnectionError( target, generation, diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index fd77985..a07d923 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -77,30 +77,13 @@ export function useLiveStream({ const [status, setStatus] = useState({ state: "idle" }); const [error, setError] = useState(""); const [fps, setFps] = useState(0); - const [pageVisible, setPageVisible] = useState( - () => document.visibilityState !== "hidden", - ); const [streamCanvasRevision, setStreamCanvasRevision] = useState(0); const [runtimeInfo] = useState(detectRuntimeInfo); - const streamPaused = paused || !pageVisible; if (!clientTelemetryIdRef.current) { clientTelemetryIdRef.current = createClientTelemetryId(); } - useEffect(() => { - const updatePageVisible = () => { - setPageVisible(document.visibilityState !== "hidden"); - }; - document.addEventListener("visibilitychange", updatePageVisible); - window.addEventListener("pageshow", updatePageVisible); - updatePageVisible(); - return () => { - document.removeEventListener("visibilitychange", updatePageVisible); - window.removeEventListener("pageshow", updatePageVisible); - }; - }, []); - useEffect(() => { let frameCount = 0; let lastSampleAt = performance.now(); @@ -126,7 +109,7 @@ export function useLiveStream({ }, []); useEffect(() => { - if (streamPaused || !canvasElement || workerClientRef.current) { + if (paused || !canvasElement || workerClientRef.current) { return; } @@ -181,7 +164,7 @@ export function useLiveStream({ workerClient.destroy(); workerClientRef.current = null; }; - }, [canvasElement, streamPaused]); + }, [canvasElement, paused]); useEffect(() => { latestDecodedFramesRef.current = stats.decodedFrames; @@ -236,7 +219,7 @@ export function useLiveStream({ setError(""); setFps(0); - if (streamPaused || !simulator?.isBooted) { + if (paused || !simulator?.isBooted) { workerClient.disconnect(); workerClient.clear(); return; @@ -254,7 +237,7 @@ export function useLiveStream({ return () => { workerClient.disconnect(); }; - }, [canvasElement, simulator?.isBooted, simulator?.udid, streamPaused]); + }, [canvasElement, simulator?.isBooted, simulator?.udid, paused]); useEffect(() => { if (!simulator?.udid) { diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index a103ecc..cc7d1e4 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -80,12 +80,6 @@ impl SimulatorSession { if matches!(*state, SessionState::Ready | SessionState::Streaming) { return Ok(()); } - if matches!(*state, SessionState::Failed) { - return Err(AppError::native(format!( - "Private simulator display attach previously failed for {}. Restart the simulator or daemon before retrying this UDID.", - self.inner.udid - ))); - } *state = SessionState::Attaching; } diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 8e6894e..9ed96e7 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -44,7 +44,6 @@ const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); const WEBRTC_REALTIME_WRITE_TIMEOUT: Duration = Duration::from_millis(45); const WEBRTC_REALTIME_KEYFRAME_WRITE_TIMEOUT: Duration = Duration::from_millis(90); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; -const WEBRTC_DISCONNECTED_GRACE: Duration = Duration::from_secs(6); static WEBRTC_MEDIA_STREAMS: OnceLock>>>> = OnceLock::new(); const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 3; @@ -160,13 +159,7 @@ pub async fn create_answer( .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, ); register_diagnostics(&peer_connection, &udid); - let (viewer_closed_token, viewer_closed) = broadcast::channel(1); - register_control_data_channel( - &peer_connection, - session.clone(), - udid.clone(), - viewer_closed_token.clone(), - ); + register_control_data_channel(&peer_connection, session.clone(), udid.clone()); let video_track = Arc::new(TrackLocalStaticRTP::new( RTCRtpCodecCapability { @@ -227,7 +220,6 @@ pub async fn create_answer( video_track, cancellation_token, cancellation, - viewer_closed, } .run(), ); @@ -339,17 +331,15 @@ fn register_control_data_channel( peer_connection: &Arc, session: crate::simulators::session::SimulatorSession, udid: String, - viewer_closed_token: broadcast::Sender<()>, ) { peer_connection.on_data_channel(Box::new(move |channel: Arc| { let session = session.clone(); let udid = udid.clone(); - let viewer_closed_token = viewer_closed_token.clone(); Box::pin(async move { if channel.label() != WEBRTC_CONTROL_CHANNEL_LABEL { return; } - attach_control_data_channel(channel, session, udid, viewer_closed_token); + attach_control_data_channel(channel, session, udid); }) })); } @@ -358,18 +348,7 @@ fn attach_control_data_channel( channel: Arc, session: crate::simulators::session::SimulatorSession, udid: String, - viewer_closed_token: broadcast::Sender<()>, ) { - let close_udid = udid.clone(); - channel.on_close(Box::new(move || { - let close_udid = close_udid.clone(); - let viewer_closed_token = viewer_closed_token.clone(); - Box::pin(async move { - info!("WebRTC control data channel closed for {close_udid}"); - let _ = viewer_closed_token.send(()); - }) - })); - channel.on_message(Box::new(move |message: DataChannelMessage| { let session = session.clone(); let udid = udid.clone(); @@ -585,7 +564,6 @@ struct WebRtcMediaStream { video_track: Arc, cancellation_token: broadcast::Sender<()>, cancellation: broadcast::Receiver<()>, - viewer_closed: broadcast::Receiver<()>, } impl WebRtcMediaStream { @@ -599,7 +577,6 @@ impl WebRtcMediaStream { video_track, cancellation_token, mut cancellation, - mut viewer_closed, } = self; let mut rx = session.subscribe(); let mut latest_keyframe = first_frame.clone(); @@ -620,7 +597,6 @@ impl WebRtcMediaStream { let mut refresh_sleep = Box::pin(time::sleep(refresh_floor)); let mut adaptive_refresh_interval = refresh_floor; let mut bootstrap_frames_remaining = WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS; - let mut disconnected_since: Option = None; let mut waiting_for_keyframe = false; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); @@ -660,25 +636,12 @@ impl WebRtcMediaStream { warn!("WebRTC media stream replaced for {udid}"); break; } - _ = viewer_closed.recv() => { - info!("WebRTC media stream closing for {udid}: control channel closed"); - break; - } _ = peer_state_interval.tick() => { let peer_state = peer_connection.connection_state(); if matches!(peer_state, RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) { warn!("WebRTC media stream closing for {udid}: peer state {peer_state}"); break; } - if matches!(peer_state, RTCPeerConnectionState::Disconnected) { - let since = disconnected_since.get_or_insert_with(time::Instant::now); - if since.elapsed() >= WEBRTC_DISCONNECTED_GRACE { - warn!("WebRTC media stream closing for {udid}: peer disconnected for {:?}", since.elapsed()); - break; - } - } else { - disconnected_since = None; - } } _ = &mut bootstrap_sleep, if bootstrap_frames_remaining > 0 => { match write_frame_sample_with_timeout( From c3794ee84174f3b58f0bfe53078c21c38baf99fe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 15:21:36 -0400 Subject: [PATCH 22/24] Clean up CI stream profile branch --- client/src/app/AppShell.tsx | 7 +- client/src/styles/components.css | 1 - docs/api/health.md | 2 +- docs/api/rest.md | 2 +- docs/cli/flags.md | 2 +- docs/guide/daemon.md | 2 +- docs/guide/lan-access.md | 2 +- docs/guide/video.md | 9 ++- server/src/api/routes.rs | 115 ------------------------------- server/src/main.rs | 2 - 10 files changed, 15 insertions(+), 129 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 2e03c88..3fbc433 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -359,7 +359,12 @@ export function AppShell() { return; } const failedUDID = selectedSimulator.udid; - setFailedStreamUDIDs((current) => new Set(current).add(failedUDID)); + setFailedStreamUDIDs((current) => { + if (current.has(failedUDID)) { + return current; + } + return new Set(current).add(failedUDID); + }); const nextSimulator = simulators.find( (simulator) => simulator.isBooted && diff --git a/client/src/styles/components.css b/client/src/styles/components.css index edf27b0..9f493ba 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1167,7 +1167,6 @@ pointer-events: none; } -.stream-image, .stream-video { position: absolute; inset: 0; diff --git a/docs/api/health.md b/docs/api/health.md index 4545b8f..3cbfb5e 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -11,7 +11,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn "ok": true, "httpPort": 4310, "timestamp": 1714094761.234, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/api/rest.md b/docs/api/rest.md index eb80eb2..351914f 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -22,7 +22,7 @@ Returns server health and the active video encoder mode. "ok": true, "httpPort": 4310, "timestamp": 1714094761.234, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/cli/flags.md b/docs/cli/flags.md index fb24221..9fcd085 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -34,7 +34,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas | `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | | `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | | `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). | +| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video Pipeline](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | | `--stream-quality` | auto/default | Optional realtime stream quality profile: `quality`, `balanced`, `smooth`, `economy`, or `ci-software`. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 98b7a45..c95e57c 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -60,7 +60,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and | `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | | `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | | `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `h264` or `h264-software`. See [Video](/guide/video). | +| `--video-codec` | `h264` | One of `h264` or `h264-software`. See [Video](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | | `--stream-quality` | auto/default | Optional realtime stream quality profile, including `ci-software` for CI providers. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/lan-access.md b/docs/guide/lan-access.md index 980f07f..ca16229 100644 --- a/docs/guide/lan-access.md +++ b/docs/guide/lan-access.md @@ -48,7 +48,7 @@ Whatever you advertise must be resolvable from the remote client. { "ok": true, "httpPort": 4310, - "videoCodec": "h264-software", + "videoCodec": "h264", "lowLatency": false, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], diff --git a/docs/guide/video.md b/docs/guide/video.md index 2d9ea5b..50f8ffb 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -8,8 +8,8 @@ The server can encode the simulator display in two modes, picked at startup with | Value | Encoder | When to use it | | --------------------------- | ------------------------------- | -------------------------------------------------------------- | -| `h264` | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | -| `h264-software` _(default)_ | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | +| `h264` _(default)_ | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | +| `h264-software` | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | Restart the daemon to change encoder mode: @@ -76,10 +76,9 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: -- **Start on the default for compatibility.** `h264-software` works without requiring the hardware encoder, but full-resolution latency can be high. -- **Switch to `h264` on local Apple Silicon when hardware encode is available.** Hardware H.264 gives the smoothest local preview with the least CPU. +- **Start on the default for local preview.** `h264` gives the smoothest preview when VideoToolbox can provide a hardware encoder. - **Switch to `h264-software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at a 640-pixel longest edge, targets 15 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. +- **Use `--stream-quality ci-software` for Studio providers on virtualized CI Macs when hardware encode is unavailable.** This profile uses software H.264 at a 960-pixel longest edge, targets 24 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. - **Use `h264-software --low-latency` only when you need the older extra-conservative software profile.** It caps at 15 fps, uses a single pending frame, reduces the longest edge to 1170 pixels, and backs off before software encode latency turns into seconds of stream delay. ## Tuning with metrics diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index b92af05..a0344cf 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -8,7 +8,6 @@ use crate::metrics::counters::{ClientStreamStats, Metrics}; use crate::native::bridge::{LogFilters, NativeBridge}; use crate::simulators::registry::SessionRegistry; use crate::simulators::session::SimulatorSession; -use crate::transport::packet::SharedFrame; use axum::body::Body; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; use axum::extract::{ConnectInfo, Path, Query, State}; @@ -17,16 +16,13 @@ use axum::middleware::{from_fn_with_state, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; -use bytes::Bytes; use futures::{SinkExt, StreamExt}; use serde::Deserialize; use serde_json::Map; use serde_json::{json as json_value, Value}; use std::collections::VecDeque; -use std::convert::Infallible; use std::env; use std::net::SocketAddr; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; @@ -429,7 +425,6 @@ pub fn router(state: AppState) -> Router { .route("/api/simulators/{udid}/touch", post(send_touch)) .route("/api/simulators/{udid}/control", get(control_socket)) .route("/api/simulators/{udid}/webrtc/offer", post(webrtc_offer)) - .route("/api/simulators/{udid}/mjpeg", get(mjpeg_stream)) .route( "/api/simulators/{udid}/touch-sequence", post(send_touch_sequence), @@ -569,7 +564,6 @@ fn normalize_video_codec(codec: &str) -> Option<&'static str> { match codec.trim().to_ascii_lowercase().as_str() { "h264" | "h264-hardware" | "avc" => Some("h264"), "h264-software" | "software-h264" => Some("h264-software"), - "jpeg" | "jpg" | "mjpeg" => Some("jpeg"), _ => None, } } @@ -1155,115 +1149,6 @@ async fn webrtc_offer( .map(Json) } -async fn mjpeg_stream( - State(state): State, - Path(udid): Path, -) -> Result { - let session = state.registry.get_or_create_async(&udid).await?; - session.ensure_started_async().await?; - session.request_refresh(); - let first_frame = session - .wait_for_keyframe(Duration::from_secs(3)) - .await - .ok_or_else(|| AppError::internal("timed out waiting for simulator JPEG frame"))?; - if first_frame.codec.as_deref() != Some("jpeg") { - return Err(AppError::bad_request( - "MJPEG stream requires starting simdeck with --video-codec jpeg", - )); - } - - let stream = futures::stream::unfold( - MjpegStreamState { - first_frame: Some(first_frame), - metrics: state.metrics.clone(), - receiver: session.subscribe(), - session, - ticker: tokio::time::interval(Duration::from_millis(67)), - }, - |mut state| async move { - state.ticker.tick().await; - state.session.request_refresh(); - let mut frame = match state.first_frame.take() { - Some(frame) => frame, - None => match state.receiver.recv().await { - Ok(frame) => frame, - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - state - .metrics - .frames_dropped_server - .fetch_add(skipped, Ordering::Relaxed); - match state.receiver.recv().await { - Ok(frame) => frame, - Err(_) => return None, - } - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => return None, - }, - }; - let mut stale_frames = 0u64; - loop { - match state.receiver.try_recv() { - Ok(next_frame) => { - stale_frames += 1; - frame = next_frame; - } - Err(tokio::sync::broadcast::error::TryRecvError::Lagged(skipped)) => { - stale_frames = stale_frames.saturating_add(skipped); - } - Err( - tokio::sync::broadcast::error::TryRecvError::Empty - | tokio::sync::broadcast::error::TryRecvError::Closed, - ) => break, - } - } - if stale_frames > 0 { - state - .metrics - .frames_dropped_server - .fetch_add(stale_frames, Ordering::Relaxed); - } - if frame.codec.as_deref() != Some("jpeg") { - return None; - } - - let payload = mjpeg_part(&frame); - state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); - Some((Ok::(Bytes::from(payload)), state)) - }, - ); - - Response::builder() - .header( - header::CONTENT_TYPE, - "multipart/x-mixed-replace; boundary=simdeck-mjpeg", - ) - .header(header::CACHE_CONTROL, "no-store, no-transform") - .header("x-accel-buffering", "no") - .body(Body::from_stream(stream)) - .map_err(|error| AppError::internal(format!("failed to build MJPEG response: {error}"))) -} - -struct MjpegStreamState { - first_frame: Option, - metrics: Arc, - receiver: tokio::sync::broadcast::Receiver, - session: SimulatorSession, - ticker: tokio::time::Interval, -} - -fn mjpeg_part(frame: &SharedFrame) -> Vec { - let header = format!( - "--simdeck-mjpeg\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\nX-SimDeck-Frame: {}\r\n\r\n", - frame.data.len(), - frame.frame_sequence - ); - let mut payload = Vec::with_capacity(header.len() + frame.data.len() + 2); - payload.extend_from_slice(header.as_bytes()); - payload.extend_from_slice(&frame.data); - payload.extend_from_slice(b"\r\n"); - payload -} - async fn handle_control_socket(state: AppState, udid: String, socket: WebSocket) { let session = match state.registry.get_or_create_async(&udid).await { Ok(session) => session, diff --git a/server/src/main.rs b/server/src/main.rs index db63894..8726d11 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -519,7 +519,6 @@ enum PasteboardCommand { enum VideoCodecMode { H264, H264Software, - Jpeg, } #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] @@ -609,7 +608,6 @@ impl VideoCodecMode { match self { Self::H264 => "h264", Self::H264Software => "h264-software", - Self::Jpeg => "jpeg", } } } From f4ae8d4b64a8a9f7363d9f2c727494077043a30d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 15:32:53 -0400 Subject: [PATCH 23/24] Fix PR CI formatting and clippy --- client/src/app/AppShell.tsx | 6 +++--- docs/guide/video.md | 8 ++++---- server/src/main.rs | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 3fbc433..55ce8f7 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -139,9 +139,9 @@ function simulatorDisplayReady(simulator: SimulatorMetadata): boolean { const display = simulator.privateDisplay; return Boolean( simulator.isBooted && - display?.displayReady && - display.displayWidth > 0 && - display.displayHeight > 0, + display?.displayReady && + display.displayWidth > 0 && + display.displayHeight > 0, ); } diff --git a/docs/guide/video.md b/docs/guide/video.md index 50f8ffb..80f96f7 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -6,10 +6,10 @@ SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video p The server can encode the simulator display in two modes, picked at startup with `--video-codec`: -| Value | Encoder | When to use it | -| --------------------------- | ------------------------------- | -------------------------------------------------------------- | -| `h264` _(default)_ | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | -| `h264-software` | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | +| Value | Encoder | When to use it | +| ------------------ | ------------------------------- | -------------------------------------------------------------- | +| `h264` _(default)_ | Hardware H.264 via VideoToolbox | Best local performance when the hardware encoder is available. | +| `h264-software` | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable. | Restart the daemon to change encoder mode: diff --git a/server/src/main.rs b/server/src/main.rs index 8726d11..bc1ec4f 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1085,10 +1085,8 @@ fn choose_daemon_port_for_bind(preferred: u16, bind: IpAddr) -> anyhow::Result bool { - if bind.is_unspecified() { - if TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_err() { - return false; - } + if bind.is_unspecified() && TcpListener::bind((Ipv4Addr::LOCALHOST, port)).is_err() { + return false; } TcpListener::bind((bind, port)).is_ok() } From 1c9afda85bbe0b70f18d060d08a0e4e1128cb68a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 1 May 2026 15:45:33 -0400 Subject: [PATCH 24/24] Use integration server for stdout screenshot test --- scripts/integration/cli.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index efc7d50..9d11c2a 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -213,10 +213,14 @@ async function main() { async () => { fs.writeFileSync( stdoutPng, - runBuffer(simdeck, ["screenshot", simulatorUDID, "--stdout"], { - timeoutMs: 300_000, - maxBuffer: 64 * 1024 * 1024, - }), + runBuffer( + simdeck, + ["--server-url", serverUrl, "screenshot", simulatorUDID, "--stdout"], + { + timeoutMs: 300_000, + maxBuffer: 64 * 1024 * 1024, + }, + ), ); assertPng(stdoutPng); },