From 92529c94c6b2b415865c957ef2c36f6c19e1e575 Mon Sep 17 00:00:00 2001 From: ao-anam Date: Mon, 30 Jun 2025 16:32:24 +0100 Subject: [PATCH] feat: add fallback video frame detection methods for safari --- src/lib/ClientMetrics.ts | 20 +++++------ src/modules/StreamingClient.ts | 63 +++++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/src/lib/ClientMetrics.ts b/src/lib/ClientMetrics.ts index 03cd9ef..2c9b71e 100644 --- a/src/lib/ClientMetrics.ts +++ b/src/lib/ClientMetrics.ts @@ -148,11 +148,11 @@ export const createRTCStatsReport = ( inboundVideo.forEach((report) => { const videoData = { - framesReceived: report.framesReceived || 'unknown', - framesDropped: report.framesDropped || 'unknown', - framesPerSecond: report.framesPerSecond || 'unknown', - packetsReceived: report.packetsReceived || 'unknown', - packetsLost: report.packetsLost || 'unknown', + framesReceived: report.framesReceived ?? 'unknown', + framesDropped: report.framesDropped ?? 'unknown', + framesPerSecond: report.framesPerSecond ?? 'unknown', + packetsReceived: report.packetsReceived ?? 'unknown', + packetsLost: report.packetsLost ?? 'unknown', resolution: report.frameWidth && report.frameHeight ? `${report.frameWidth}x${report.frameHeight}` @@ -172,9 +172,9 @@ export const createRTCStatsReport = ( inboundAudio.forEach((report) => { const audioData = { - packetsReceived: report.packetsReceived || 'unknown', - packetsLost: report.packetsLost || 'unknown', - audioLevel: report.audioLevel || 'unknown', + packetsReceived: report.packetsReceived ?? 'unknown', + packetsLost: report.packetsLost ?? 'unknown', + audioLevel: report.audioLevel ?? 'unknown', jitter: report.jitter !== undefined ? report.jitter : undefined, totalAudioEnergy: report.totalAudioEnergy !== undefined @@ -194,8 +194,8 @@ export const createRTCStatsReport = ( outboundAudio.forEach((report) => { const userAudioData = { - packetsSent: report.packetsSent || 'unknown', - retransmittedPackets: report.retransmittedPacketsSent || undefined, + packetsSent: report.packetsSent ?? 'unknown', + retransmittedPackets: report.retransmittedPacketsSent ?? undefined, avgPacketSendDelay: report.totalPacketSendDelay !== undefined ? (report.totalPacketSendDelay / (report.packetsSent || 1)) * 1000 diff --git a/src/modules/StreamingClient.ts b/src/modules/StreamingClient.ts index 66f63f0..a9df6c7 100644 --- a/src/modules/StreamingClient.ts +++ b/src/modules/StreamingClient.ts @@ -146,23 +146,50 @@ export class StreamingClient { try { const stats = await this.peerConnection.getStats(); + + let videoDetected = false; + let detectionMethod = null; + stats.forEach((report) => { // Find the report for inbound video if (report.type === 'inbound-rtp' && report.kind === 'video') { - if (report.framesReceived > 0) { - this.successMetricFired = true; - sendClientMetric( - ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS, - '1', - ); - if (this.successMetricPoller) { - clearInterval(this.successMetricPoller); - } - clearTimeout(timeoutId); - this.successMetricPoller = null; + // Method 1: Try framesDecoded (most reliable when available) + if ( + report.framesDecoded !== undefined && + report.framesDecoded > 0 + ) { + videoDetected = true; + detectionMethod = 'framesDecoded'; + } else if ( + report.framesReceived !== undefined && + report.framesReceived > 0 + ) { + videoDetected = true; + detectionMethod = 'framesReceived'; + } else if ( + report.bytesReceived > 0 && + report.packetsReceived > 0 && + // Additional check: ensure we've received enough data for actual video + report.bytesReceived > 100000 // rough threshold + ) { + videoDetected = true; + detectionMethod = 'bytesReceived'; } } }); + if (videoDetected && !this.successMetricFired) { + this.successMetricFired = true; + sendClientMetric( + ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS, + '1', + detectionMethod ? { detectionMethod } : undefined, + ); + if (this.successMetricPoller) { + clearInterval(this.successMetricPoller); + } + clearTimeout(timeoutId); + this.successMetricPoller = null; + } } catch (error) {} }, 500); } @@ -448,6 +475,14 @@ export class StreamingClient { // unregister the callback after the first frame this.videoElement?.cancelVideoFrameCallback(handle); this.publicEventEmitter.emit(AnamEvent.VIDEO_PLAY_STARTED); + if (!this.successMetricFired) { + this.successMetricFired = true; + sendClientMetric( + ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS, + '1', + { detectionMethod: 'videoElement' }, + ); + } }); } } else if (event.track.kind === 'audio') { @@ -568,11 +603,7 @@ export class StreamingClient { private async shutdown() { if (this.showPeerConnectionStatsReport) { const stats = await this.peerConnection?.getStats(); - if (!stats) { - console.error( - 'StreamingClient - shutdown: peer connection is unavailable. Unable to create RTC stats report.', - ); - } else { + if (stats) { const report = createRTCStatsReport( stats, this.peerConnectionStatsReportOutputFormat,