From 35033bb2db1ca630f4f7895e7678bd0ee6cfd9ef Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 13 Dec 2022 10:14:17 -0800 Subject: [PATCH] fix(HLS): Fix AV sync over ad boundaries (#4824) If server-side ad segments aren't aligned, AV could get out of sync by accumulating errors in the timestampOffset of the SourceBuffers. This improves the issue by tracking discontinuity boundaries and resetting timestampOffset to theoretical segment start times when a boundary is crossed. Issue #4589 --- lib/hls/hls_parser.js | 16 ++++++++++++---- lib/media/media_source_engine.js | 16 ++++++++++++++++ lib/media/segment_reference.js | 3 +++ lib/media/streaming_engine.js | 15 +++++++++++++-- test/test/util/manifest_parser_util.js | 7 ++++--- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 037281d45e..832b710a4b 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -2603,12 +2603,12 @@ shaka.hls.HlsParser = class { /** @type {shaka.extern.HlsAes128Key|undefined} */ let hlsAes128Key = undefined; - // We may need to look at the media itself to determine a segment start - // time. + let discontinuitySequence = shaka.hls.Utils.getFirstTagWithNameAsNumber( + playlist.tags, 'EXT-X-DISCONTINUITY-SEQUENCE', 0); const mediaSequenceNumber = shaka.hls.Utils.getFirstTagWithNameAsNumber( playlist.tags, 'EXT-X-MEDIA-SEQUENCE', 0); - const skipTag = shaka.hls.Utils.getFirstTagWithName(playlist.tags, - 'EXT-X-SKIP'); + const skipTag = shaka.hls.Utils.getFirstTagWithName( + playlist.tags, 'EXT-X-SKIP'); const skippedSegments = skipTag ? Number(skipTag.getAttributeValue('SKIPPED-SEGMENTS')) : 0; let position = mediaSequenceNumber + skippedSegments; @@ -2631,6 +2631,12 @@ shaka.hls.HlsParser = class { (i == 0) ? firstStartTime : previousReference.endTime; position = mediaSequenceNumber + skippedSegments + i; + const discontinuityTag = shaka.hls.Utils.getFirstTagWithName( + playlist.tags, 'EXT-X-DISCONTINUITY'); + if (discontinuityTag) { + discontinuitySequence++; + } + // Apply new AES-128 tags as you see them, keeping a running total. for (const drmTag of item.tags) { if (drmTag.name == 'EXT-X-KEY' && @@ -2668,6 +2674,8 @@ shaka.hls.HlsParser = class { previousReference = reference; if (reference) { + reference.discontinuitySequence = discontinuitySequence; + if (this.config_.hls.ignoreManifestProgramDateTime && this.minSequenceNumber_ != null && position < this.minSequenceNumber_) { diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index c2a1f123b2..50ac6a7a55 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -896,6 +896,22 @@ shaka.media.MediaSourceEngine = class { ]); } + /** + * Adjust timestamp offset to maintain AV sync across discontinuities. + * Only used in sequence mode. + * + * @param {shaka.util.ManifestParserUtils.ContentType} contentType + * @param {number} timestampOffset + * @return {!Promise} + */ + async resync(contentType, timestampOffset) { + goog.asserts.assert(this.sequenceMode_, + 'resyncAudio only used with sequence mode!'); + await this.enqueueOperation_( + contentType, + () => this.setTimestampOffset_(contentType, timestampOffset)); + } + /** * @param {string=} reason Valid reasons are 'network' and 'decode'. * @return {!Promise} diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index b74a996aa2..38d1851504 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -246,6 +246,9 @@ shaka.media.SegmentReference = class { /** @type {?shaka.media.SegmentReference.ThumbnailSprite} */ this.thumbnailSprite = null; + + /** @type {number} */ + this.discontinuitySequence = 0; } /** diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index 295f002edd..a319f5bf79 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1659,6 +1659,19 @@ shaka.media.StreamingEngine = class { } } + if (this.manifest_.sequenceMode) { + // Across discontinuity bounds, we should resync timestamps for + // sequence mode playbacks. The next segment appended should + // land at its theoretical timestamp from the segment index. + const lastDiscontinuitySequence = + mediaState.lastSegmentReference ? + mediaState.lastSegmentReference.discontinuitySequence : null; + if (reference.discontinuitySequence != lastDiscontinuitySequence) { + operations.push(this.playerInterface_.mediaSourceEngine.resync( + mediaState.type, reference.startTime)); + } + } + await Promise.all(operations); } @@ -1713,8 +1726,6 @@ shaka.media.StreamingEngine = class { await this.evict_(mediaState, presentationTime); this.destroyer_.ensureNotDestroyed(); - shaka.log.v1(logPrefix, 'appending media segment at', - (reference.syncTime == null ? 'unknown' : reference.syncTime)); // 'seeked' or 'adaptation' triggered logic applies only to this // appendBuffer() call. diff --git a/test/test/util/manifest_parser_util.js b/test/test/util/manifest_parser_util.js index a743ae75b5..8e5c22fd85 100644 --- a/test/test/util/manifest_parser_util.js +++ b/test/test/util/manifest_parser_util.js @@ -73,7 +73,7 @@ shaka.test.ManifestParser = class { const appendWindowStart = /** @type {?} */(jasmine.any(Number)); const appendWindowEnd = /** @type {?} */(jasmine.any(Number)); - return new shaka.media.SegmentReference( + const ref = new shaka.media.SegmentReference( start, end, getUris, startByte, endByte, initSegmentReference, timestampOffset, @@ -82,7 +82,8 @@ shaka.test.ManifestParser = class { partialReferences, tilesLayout, /* tileDuration= */ undefined, - syncTime, - ); + syncTime); + ref.discontinuitySequence = /** @type {?} */(jasmine.any(Number)); + return ref; } };