Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ more important than full-resolution smoothness:
simdeck daemon start --video-codec software --low-latency
```

Local browser streams default to 60 fps. On high-refresh local displays, opt in
to a paced hardware stream up to 120 fps:

```sh
simdeck daemon restart --local-stream-fps 120
```

Restart the CoreSimulator service layer when `simctl` reports a stale service
version or the live display gets stuck before the first frame:

Expand Down
112 changes: 67 additions & 45 deletions cli/XCWH264Encoder.m
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
static const int32_t XCWTargetRealTimeFrameRate = 60;
static const int32_t XCWTargetRealtimeHardwareFrameRate = 30;
static const int32_t XCWTargetSoftwareFrameRate = 60;
static const int32_t XCWMinimumLocalStreamFrameRate = 15;
static const int32_t XCWMaximumLocalStreamFrameRate = 120;
static const int32_t XCWTargetLowLatencySoftwareFrameRate = 15;
static const NSUInteger XCWMaximumInFlightFrames = 2;
static const int32_t XCWMinimumAverageBitRate = 18000000;
Expand Down Expand Up @@ -137,6 +139,22 @@ static uint64_t XCWRealtimeMaximumFrameIntervalUs(void) {
return MAX(XCWRealtimeFrameIntervalUs() * 2, XCWRealtimeFrameIntervalUs());
}

static int32_t XCWLocalStreamTargetFrameRate(void) {
return XCWIntFromEnvironment(@"SIMDECK_LOCAL_STREAM_FPS",
XCWTargetRealTimeFrameRate,
XCWMinimumLocalStreamFrameRate,
XCWMaximumLocalStreamFrameRate);
}

static uint64_t XCWLocalStreamFrameIntervalUs(void) {
int32_t fps = MAX(1, XCWLocalStreamTargetFrameRate());
return (uint64_t)llround(1000000.0 / (double)fps);
}

static uint64_t XCWLocalStreamMaximumFrameIntervalUs(void) {
return MAX(XCWSoftwareMaximumFrameIntervalUs, XCWLocalStreamFrameIntervalUs());
}

static int64_t XCWRealtimeBitsPerPixelBudgetValue(void) {
return XCWInt64FromEnvironment(@"SIMDECK_REALTIME_BITS_PER_PIXEL",
XCWRealtimeBitsPerPixelBudget,
Expand Down Expand Up @@ -490,10 +508,10 @@ @implementation XCWH264Encoder {
uint64_t _lastSoftwareSubmissionUs;
NSUInteger _softwarePacedFrameCount;
NSUInteger _softwareHealthyFrameCount;
uint64_t _realtimeHardwareFrameIntervalUs;
uint64_t _lastRealtimeHardwareSubmissionUs;
NSUInteger _realtimeHardwarePacedFrameCount;
NSUInteger _realtimeHardwareHealthyFrameCount;
uint64_t _hardwareFrameIntervalUs;
uint64_t _lastHardwareSubmissionUs;
NSUInteger _hardwarePacedFrameCount;
NSUInteger _hardwareHealthyFrameCount;
NSString *_selectedEncoderID;
NSInteger _lastSessionStatus;
NSInteger _lastPrepareStatus;
Expand All @@ -518,7 +536,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler
_realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode;
_codecType = XCWVideoCodecTypeForMode(_encoderMode);
_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
return self;
}

Expand Down Expand Up @@ -568,9 +586,9 @@ - (void)reconfigureForStreamQualityChange {
self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked];
self->_softwarePacedFrameCount = 0;
self->_softwareHealthyFrameCount = 0;
self->_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
self->_realtimeHardwarePacedFrameCount = 0;
self->_realtimeHardwareHealthyFrameCount = 0;
self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
self->_hardwarePacedFrameCount = 0;
self->_hardwareHealthyFrameCount = 0;
});
}

Expand All @@ -597,9 +615,13 @@ - (NSDictionary *)statsRepresentation {
@"softwareFrameIntervalUs": @(self->_softwareFrameIntervalUs),
@"softwareTargetFps": @(self->_softwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_softwareFrameIntervalUs) : 0.0),
@"softwarePacedFrames": @(self->_softwarePacedFrameCount),
@"realtimeHardwareFrameIntervalUs": @(self->_realtimeHardwareFrameIntervalUs),
@"realtimeHardwareTargetFps": @(self->_realtimeHardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_realtimeHardwareFrameIntervalUs) : 0.0),
@"realtimeHardwarePacedFrames": @(self->_realtimeHardwarePacedFrameCount),
@"localStreamTargetFps": @(XCWLocalStreamTargetFrameRate()),
@"hardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs),
@"hardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0),
@"hardwarePacedFrames": @(self->_hardwarePacedFrameCount),
@"realtimeHardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs),
@"realtimeHardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0),
@"realtimeHardwarePacedFrames": @(self->_hardwarePacedFrameCount),
@"transportCodec": XCWCodecName(self->_codecType),
@"encoderMode": XCWVideoEncoderModeName(self->_encoderMode),
@"lowLatencyMode": @(self->_lowLatencyMode),
Expand Down Expand Up @@ -665,15 +687,15 @@ - (NSUInteger)softwareHealthyFrameWindowLocked {
}

- (uint64_t)minimumHardwareFrameIntervalUsLocked {
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWSoftwareMinimumFrameIntervalUs;
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs();
}

- (uint64_t)initialHardwareFrameIntervalUsLocked {
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWSoftwareInitialFrameIntervalUs;
return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs();
}

- (uint64_t)maximumHardwareFrameIntervalUsLocked {
return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWSoftwareMaximumFrameIntervalUs;
return _realtimeStreamMode ? XCWRealtimeMaximumFrameIntervalUs() : XCWLocalStreamMaximumFrameIntervalUs();
}

- (int32_t)expectedFrameRateLocked {
Expand All @@ -686,24 +708,24 @@ - (int32_t)expectedFrameRateLocked {
if (_realtimeStreamMode) {
return XCWRealtimeTargetFrameRate();
}
return XCWTargetRealTimeFrameRate;
return XCWLocalStreamTargetFrameRate();
}

- (BOOL)shouldPaceRealtimeHardwareFrameAtTimeUs:(uint64_t)nowUs {
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || _needsKeyFrame) {
- (BOOL)shouldPaceHardwareFrameAtTimeUs:(uint64_t)nowUs {
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || _needsKeyFrame) {
return NO;
}
if (_realtimeHardwareFrameIntervalUs == 0) {
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
if (_hardwareFrameIntervalUs == 0) {
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
}
if (_lastRealtimeHardwareSubmissionUs == 0) {
if (_lastHardwareSubmissionUs == 0) {
return NO;
}
uint64_t elapsedUs = nowUs >= _lastRealtimeHardwareSubmissionUs ? nowUs - _lastRealtimeHardwareSubmissionUs : 0;
if (elapsedUs >= _realtimeHardwareFrameIntervalUs) {
uint64_t elapsedUs = nowUs >= _lastHardwareSubmissionUs ? nowUs - _lastHardwareSubmissionUs : 0;
if (elapsedUs >= _hardwareFrameIntervalUs) {
return NO;
}
_realtimeHardwarePacedFrameCount += 1;
_hardwarePacedFrameCount += 1;
return YES;
}

Expand Down Expand Up @@ -764,41 +786,41 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs {
_softwareHealthyFrameCount = 0;
}

- (void)adaptRealtimeHardwarePacingForLatencyUs:(uint64_t)latencyUs {
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) {
- (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs {
if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || latencyUs == 0) {
return;
}
if (_realtimeHardwareFrameIntervalUs == 0) {
_realtimeHardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
if (_hardwareFrameIntervalUs == 0) {
_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked];
}

uint64_t minimumIntervalUs = [self minimumHardwareFrameIntervalUsLocked];
uint64_t maximumIntervalUs = [self maximumHardwareFrameIntervalUsLocked];
if (latencyUs > _realtimeHardwareFrameIntervalUs) {
uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs;
if (latencyUs > _hardwareFrameIntervalUs) {
uint64_t nextIntervalUs = _hardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs;
uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs;
if (nextIntervalUs < latencyBoundIntervalUs) {
nextIntervalUs = latencyBoundIntervalUs;
}
_realtimeHardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs);
_realtimeHardwareHealthyFrameCount = 0;
_hardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs);
_hardwareHealthyFrameCount = 0;
return;
}

if (latencyUs < _realtimeHardwareFrameIntervalUs &&
_realtimeHardwareFrameIntervalUs > minimumIntervalUs) {
_realtimeHardwareHealthyFrameCount += 1;
if (_realtimeHardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) {
uint64_t nextIntervalUs = _realtimeHardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs
? _realtimeHardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs
if (latencyUs < _hardwareFrameIntervalUs &&
_hardwareFrameIntervalUs > minimumIntervalUs) {
_hardwareHealthyFrameCount += 1;
if (_hardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) {
uint64_t nextIntervalUs = _hardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs
? _hardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs
: minimumIntervalUs;
_realtimeHardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs);
_realtimeHardwareHealthyFrameCount = 0;
_hardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs);
_hardwareHealthyFrameCount = 0;
}
return;
}

_realtimeHardwareHealthyFrameCount = 0;
_hardwareHealthyFrameCount = 0;
}

- (void)drainPendingFramesLocked {
Expand Down Expand Up @@ -839,7 +861,7 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
}

uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceRealtimeHardwareFrameAtTimeUs:nowUs]) {
if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceHardwareFrameAtTimeUs:nowUs]) {
return YES;
}

Expand Down Expand Up @@ -885,8 +907,8 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer {
_submittedFrameCount += 1;
if (_encoderMode == XCWVideoEncoderModeH264Software) {
_lastSoftwareSubmissionUs = nowUs;
} else if ((_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) && _realtimeStreamMode) {
_lastRealtimeHardwareSubmissionUs = nowUs;
} else if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) {
_lastHardwareSubmissionUs = nowUs;
}
_maxInFlightFrameCount = MAX(_maxInFlightFrameCount, _inFlightFrameCount);
if (_encoderMode == XCWVideoEncoderModeH264Software || !_realtimeStreamMode) {
Expand Down Expand Up @@ -1013,7 +1035,7 @@ - (void)invalidateCompressionSessionLocked {
_timestampOriginUs = 0;
_inFlightFrameCount = 0;
_lastSoftwareSubmissionUs = 0;
_lastRealtimeHardwareSubmissionUs = 0;
_lastHardwareSubmissionUs = 0;
_hardwareAccelerated = NO;
_selectedEncoderID = nil;
[self invalidateScalingResourcesLocked];
Expand Down Expand Up @@ -1203,7 +1225,7 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer
uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0);
_latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0;
[self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs];
[self adaptRealtimeHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
[self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs];
}
NSString *codec = nil;
NSData *decoderConfig = nil;
Expand Down
1 change: 1 addition & 0 deletions client/src/features/stream/streamTypes.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { Size } from "../viewport/types";

export interface StreamConnectTarget {
clientId?: string;
remote?: boolean;
udid: string;
}
Expand Down
43 changes: 32 additions & 11 deletions client/src/features/stream/streamWorkerClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ const HAVE_CURRENT_DATA = 2;
const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control";
const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry";
const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000;
const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000;
const WEBRTC_DISCONNECTED_GRACE_MS = 8000;
const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 15000;
const WEBRTC_REMOTE_RECEIVER_BUFFER_SECONDS = 0.06;
const WEBRTC_DISCONNECTED_GRACE_MS = 30000;
const WEBRTC_RECONNECT_BASE_DELAY_MS = 3000;
const WEBRTC_RECONNECT_MAX_DELAY_MS = 10000;

Expand Down Expand Up @@ -47,9 +48,9 @@ function sendDataChannelMessage(

export function buildStreamTarget(
udid: string,
options: { remote?: boolean } = {},
options: { clientId?: string; remote?: boolean } = {},
): StreamConnectTarget {
return { remote: options.remote, udid };
return { clientId: options.clientId, remote: options.remote, udid };
}

export function canUseWebRtc(): boolean {
Expand Down Expand Up @@ -145,7 +146,10 @@ class WebRtcStreamClient implements StreamClientBackend {
direction: "recvonly",
});
configureReceiverCodecPreferences(transceiver);
configureLowLatencyReceiver(transceiver.receiver);
configureLowLatencyReceiver(
transceiver.receiver,
target.remote ? WEBRTC_REMOTE_RECEIVER_BUFFER_SECONDS : null,
);
const controlChannel = peerConnection.createDataChannel(
WEBRTC_CONTROL_CHANNEL_LABEL,
{
Expand Down Expand Up @@ -180,7 +184,10 @@ class WebRtcStreamClient implements StreamClientBackend {
}
event.track.contentHint = "motion";
for (const receiver of peerConnection.getReceivers()) {
configureLowLatencyReceiver(receiver);
configureLowLatencyReceiver(
receiver,
target.remote ? WEBRTC_REMOTE_RECEIVER_BUFFER_SECONDS : null,
);
}
const stream = event.streams[0] ?? new MediaStream([event.track]);
const video = document.createElement("video");
Expand Down Expand Up @@ -431,14 +438,22 @@ class WebRtcStreamClient implements StreamClientBackend {
const hasRenderedFrame = this.stats.renderedFrames > 0;
const frameAgeMs =
this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity;
if (!hasRenderedFrame || frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) {
if (!hasRenderedFrame) {
this.handleConnectionError(
target,
generation,
new Error("WebRTC video stalled before rendering fresh frames."),
);
return;
}
if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) {
this.handleConnectionError(
target,
generation,
new Error("WebRTC video stalled after rendering frames."),
);
return;
}
this.scheduleFrameWatchdog(target, generation);
},
this.stats.renderedFrames > 0
Expand Down Expand Up @@ -575,7 +590,7 @@ class WebRtcStreamClient implements StreamClientBackend {
private postDiagnostics(target: StreamConnectTarget, detail: string) {
const payload = {
...this.stats,
clientId: "webrtc-page",
clientId: target.clientId ?? "webrtc-page",
connectionId: this.connectGeneration,
detail,
iceConnectionState: this.diagnostics.iceConnectionState,
Expand Down Expand Up @@ -778,16 +793,22 @@ function postWebRtcOffer(
);
}

function configureLowLatencyReceiver(receiver: RTCRtpReceiver) {
function configureLowLatencyReceiver(
receiver: RTCRtpReceiver,
bufferSeconds: number | null,
) {
if (!bufferSeconds || bufferSeconds <= 0) {
return;
}
const lowLatencyReceiver = receiver as RTCRtpReceiver & {
jitterBufferTarget?: number;
playoutDelayHint?: number;
};
if ("jitterBufferTarget" in lowLatencyReceiver) {
lowLatencyReceiver.jitterBufferTarget = 0.001;
lowLatencyReceiver.jitterBufferTarget = bufferSeconds;
}
if ("playoutDelayHint" in lowLatencyReceiver) {
lowLatencyReceiver.playoutDelayHint = 0.001;
lowLatencyReceiver.playoutDelayHint = bufferSeconds;
}
}

Expand Down
7 changes: 6 additions & 1 deletion client/src/features/stream/useLiveStream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,12 @@ export function useLiveStream({
return;
}

workerClient.connect(buildStreamTarget(simulator.udid, { remote }));
workerClient.connect(
buildStreamTarget(simulator.udid, {
clientId: clientTelemetryIdRef.current,
remote,
}),
);
return () => {
workerClient.disconnect();
};
Expand Down
1 change: 1 addition & 0 deletions docs/api/health.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ Returns the static bootstrap information the browser client needs, plus a freshn
| `videoCodec` | Requested encoder mode. One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). |
| `lowLatency` | `true` when software H.264 low-latency mode was enabled at daemon startup. |
| `realtimeStream` | `true` when the WebRTC stream is configured to favor freshness and realtime pacing. |
| `localStreamFps` | Local quality stream frame cap, from 15 to 120 fps. Defaults to 60. |
| `streamQuality` | Active realtime quality profile and encoder limits such as `maxEdge`, `fps`, and bitrate. |
| `webRtc.iceServers` | ICE servers the browser should use when creating the WebRTC peer connection. |
| `webRtc.iceTransportPolicy` | Browser ICE transport policy. One of `all` or `relay`. |
Expand Down
Loading
Loading