From 629a21dbe7b97eab04a61a08e39e76cf5263c352 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 13 Jun 2022 13:35:57 -0700 Subject: [PATCH] fix(hls): Fix AV sync issues, fallback to sequence numbers if PROGRAM-DATE-TIME ignored (#4289) We now have an explicit fallback to sync on sequence numbers if PROGRAM-DATE-TIME is explicitly ignored. This is more robust than relying on whatever happens to be first in the various media playlists. This also refactors how PROGRAM-DATE-TIME is used. The date will now be used to adjust segment reference start times, rather than overriding the time used in StreamingEngine. (This was hard to discover when reading the HLS parser.) Now all HLS sync logic is in the HLS parser. Closes #4287 --- externs/shaka/player.js | 6 +- lib/hls/hls_parser.js | 233 ++++++++++++++++++++++++--------- lib/media/segment_index.js | 27 +++- lib/media/segment_reference.js | 13 +- lib/media/streaming_engine.js | 2 +- test/hls/hls_parser_unit.js | 110 ++++++++++++++-- 6 files changed, 312 insertions(+), 79 deletions(-) diff --git a/externs/shaka/player.js b/externs/shaka/player.js index bb7b299554..1f38b2399d 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -766,8 +766,10 @@ shaka.extern.DashManifestConfiguration; * Defaults to 'avc1.42E01E'. * @property {boolean} ignoreManifestProgramDateTime * If true, the HLS parser will ignore the - * EXT-X-PROGRAM-DATE-TIME tags in the manifest. - * Meant for tags that are incorrect or malformed. + * EXT-X-PROGRAM-DATE-TIME tags in the manifest and use media + * sequence numbers instead. + * Meant for streams where EXT-X-PROGRAM-DATE-TIME is incorrect + * or malformed. * Defaults to false. * @property {string} mediaPlaylistFullMimeType * A string containing a full mime type, including both the basic mime type diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index d1af081360..742bc13386 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -121,13 +121,19 @@ shaka.hls.HlsParser = class { this.updatePlaylistDelay_ = 0; /** - * A time offset to apply to EXT-X-PROGRAM-DATE-TIME values to normalize - * them so that they start at 0. This is necessary because these times will - * be used to set presentation times for segments. - * null means we don't have enough data yet. + * If true, we have already calculated offsets to synchronize streams. + * Offsets are computed in syncStreams*_(). + * @private {boolean} + */ + this.streamsSynced_ = false; + + /** + * The minimum sequence number for generated segments, when ignoring + * EXT-X-PROGRAM-DATE-TIME. + * * @private {?number} */ - this.syncTimeOffset_ = null; + this.minSequenceNumber_ = null; /** * This timer is used to trigger the start of a manifest update. A manifest @@ -361,39 +367,155 @@ shaka.hls.HlsParser = class { } /** - * If necessary, makes sure that sync times will be normalized to 0, so that - * a stream does not start buffering at 50 years in because sync times are - * measured in time since 1970. + * Align all streams by sequence number by dropping early segments. Then + * offset all streams to begin at presentation time 0. * @private */ - calculateSyncTimeOffset_() { - if (this.syncTimeOffset_ != null) { - // The offset was already calculated. + syncStreamsWithSequenceNumber_() { + if (this.streamsSynced_) { return; } - const segments = new Set(); - let lowestSyncTime = Infinity; + // Sync using media sequence number. Find the highest starting sequence + // number among all streams. Later, we will drop any references to + // earlier segments in other streams, then offset everything back to 0. + let highestStartingSequenceNumber = -1; + const firstSequenceNumberMap = new Map(); + for (const streamInfo of this.uriToStreamInfosMap_.values()) { const segmentIndex = streamInfo.stream.segmentIndex; if (segmentIndex) { - segmentIndex.forEachTopLevelReference((segment) => { - if (segment.syncTime != null) { - lowestSyncTime = Math.min(lowestSyncTime, segment.syncTime); - segments.add(segment); + const segment0 = segmentIndex.earliestReference(); + if (segment0) { + // This looks inefficient, but iteration order is insertion order. + // So the very first entry should be the one we want. + // We assert that this holds true so that we are alerted by debug + // builds and tests if it changes. We still do a loop, though, so + // that the code functions correctly in production no matter what. + if (goog.DEBUG) { + const firstSequenceStartTime = + streamInfo.mediaSequenceToStartTime.values().next().value; + goog.asserts.assert( + firstSequenceStartTime == segment0.startTime, + 'Sequence number map is not ordered as expected!'); + } + for (const [sequence, start] of streamInfo.mediaSequenceToStartTime) { + if (start == segment0.startTime) { + firstSequenceNumberMap.set(streamInfo, sequence); + + highestStartingSequenceNumber = Math.max( + highestStartingSequenceNumber, sequence); + break; + } } - }); + } + } + } + + if (highestStartingSequenceNumber < 0) { + // Nothing to sync. + return; + } + + // From now on, updates will ignore any references before this number. + this.minSequenceNumber_ = highestStartingSequenceNumber; + + shaka.log.debug('Syncing HLS streams against base sequence number:', + this.minSequenceNumber_); + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex) { + // Drop any earlier references. + const numSegmentsToDrop = this.minSequenceNumber_ - + firstSequenceNumberMap.get(streamInfo); + segmentIndex.dropFirstReferences(numSegmentsToDrop); + + // Now adjust timestamps back to begin at 0. + const segmentN = segmentIndex.earliestReference(); + if (segmentN) { + this.offsetStream_(streamInfo, -segmentN.startTime); + } + } + } + + this.streamsSynced_ = true; + } + + /** + * Synchronize streams by the EXT-X-PROGRAM-DATE-TIME tags attached to their + * segments. Also normalizes segment times so that the earliest segment in + * any stream is at time 0. + * @private + */ + syncStreamsWithProgramDateTime_() { + if (this.streamsSynced_) { + return; + } + + let lowestSyncTime = Infinity; + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex) { + const segment0 = segmentIndex.earliestReference(); + if (segment0 != null && segment0.syncTime != null) { + lowestSyncTime = Math.min(lowestSyncTime, segment0.syncTime); + } } } - if (segments.size > 0) { - this.syncTimeOffset_ = -lowestSyncTime; - for (const segment of segments) { - segment.syncTime += this.syncTimeOffset_; - for (const partial of segment.partialReferences) { - partial.syncTime += this.syncTimeOffset_; + + if (lowestSyncTime == Infinity) { + // Nothing to sync. + return; + } + + shaka.log.debug('Syncing HLS streams against base time:', lowestSyncTime); + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex != null) { + const segment0 = segmentIndex.earliestReference(); + if (segment0.syncTime == null) { + shaka.log.alwaysError('Missing EXT-X-PROGRAM-DATE-TIME for stream', + streamInfo.verbatimMediaPlaylistUri, + 'Expect AV sync issues!'); + } else { + // The first segment's target startTime should be based entirely on + // its syncTime. The rest of the stream will be based on that + // starting point. The earliest segment sync time from any stream + // will become presentation time 0. If two streams start e.g. 6 + // seconds apart in syncTime, then their first segments will also + // start 6 seconds apart in presentation time. + const segment0TargetTime = segment0.syncTime - lowestSyncTime; + const streamOffset = segment0TargetTime - segment0.startTime; + + this.offsetStream_(streamInfo, streamOffset); } } } + + this.streamsSynced_ = true; + } + + /** + * @param {!shaka.hls.HlsParser.StreamInfo} streamInfo + * @param {number} offset + * @private + */ + offsetStream_(streamInfo, offset) { + streamInfo.stream.segmentIndex.offset(offset); + + streamInfo.maxTimestamp += offset; + goog.asserts.assert(streamInfo.maxTimestamp >= 0, + 'Negative maxTimestamp after adjustment!'); + + for (const [key, value] of streamInfo.mediaSequenceToStartTime) { + streamInfo.mediaSequenceToStartTime.set(key, value + offset); + } + + shaka.log.debug('Offset', offset, 'applied to', + streamInfo.verbatimMediaPlaylistUri); } /** @@ -524,7 +646,11 @@ shaka.hls.HlsParser = class { // Now that we have generated all streams, we can determine the offset to // apply to sync times. - this.calculateSyncTimeOffset_(); + if (this.config_.hls.ignoreManifestProgramDateTime) { + this.syncStreamsWithSequenceNumber_(); + } else { + this.syncStreamsWithProgramDateTime_(); + } if (this.aesEncrypted_ && variants.length == 0) { // We do not support AES-128 encryption with HLS yet. Variants is null @@ -539,15 +665,12 @@ shaka.hls.HlsParser = class { // Find the min and max timestamp of the earliest segment in all streams. // Find the minimum duration of all streams as well. - let minFirstTimestamp = Infinity; let minDuration = Infinity; - for (const streamInfo of this.uriToStreamInfosMap_.values()) { - minFirstTimestamp = - Math.min(minFirstTimestamp, streamInfo.minTimestamp); if (streamInfo.stream.type != 'text') { - minDuration = Math.min(minDuration, - streamInfo.maxTimestamp - streamInfo.minTimestamp); + // Since everything is already offset to 0 (either by sync or by being + // VOD), only maxTimestamp is necessary to compute the duration. + minDuration = Math.min(minDuration, streamInfo.maxTimestamp); } } @@ -591,18 +714,11 @@ shaka.hls.HlsParser = class { segmentAvailabilityDuration); } } else { - // For VOD/EVENT content, offset everything back to 0. - // Use the minimum timestamp as the offset for all streams. // Use the minimum duration as the presentation duration. this.presentationTimeline_.setDuration(minDuration); - // Use a negative offset to adjust towards 0. - this.presentationTimeline_.offset(-minFirstTimestamp); for (const streamInfo of this.uriToStreamInfosMap_.values()) { - // The segments were created with actual media times, rather than - // presentation-aligned times, so offset them all now. - streamInfo.stream.segmentIndex.offset(-minFirstTimestamp); - // Finally, fit the segments to the playlist duration. + // Fit the segments to the playlist duration. streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration); } } @@ -810,12 +926,7 @@ shaka.hls.HlsParser = class { }); // Create stream info for each audio / video media tag. - // Wait for the first stream info created, so that the start time is fetched - // and can be reused. - if (mediaTags.length) { - await this.createStreamInfoFromMediaTag_(mediaTags[0]); - } - const promises = mediaTags.slice(1).map((tag) => { + const promises = mediaTags.map((tag) => { return this.createStreamInfoFromMediaTag_(tag); }); await Promise.all(promises); @@ -1583,7 +1694,6 @@ shaka.hls.HlsParser = class { throw error; } - const minTimestamp = segments[0].startTime; const lastEndTime = segments[segments.length - 1].endTime; /** @type {!shaka.media.SegmentIndex} */ const segmentIndex = new shaka.media.SegmentIndex(segments); @@ -1641,7 +1751,6 @@ shaka.hls.HlsParser = class { stream, verbatimMediaPlaylistUri, absoluteMediaPlaylistUri, - minTimestamp, maxTimestamp: lastEndTime, mediaSequenceToStartTime, canSkipSegments, @@ -1853,15 +1962,9 @@ shaka.hls.HlsParser = class { const dateTimeTag = shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-PROGRAM-DATE-TIME'); if (dateTimeTag && dateTimeTag.value) { - const time = shaka.util.XmlUtils.parseDate(dateTimeTag.value); - goog.asserts.assert(time != null, + syncTime = shaka.util.XmlUtils.parseDate(dateTimeTag.value); + goog.asserts.assert(syncTime != null, 'EXT-X-PROGRAM-DATE-TIME format not valid'); - // Sync time offset is null on the first go-through. This indicates that - // we have not yet seen every stream, and thus do not yet have enough - // information to determine how to normalize the sync times. - // For that first go-through, the sync time will be applied after the - // references are all created. Until then, just offset by 0. - syncTime = time + (this.syncTimeOffset_ || 0); } } @@ -2085,16 +2188,13 @@ shaka.hls.HlsParser = class { firstStartTime = mediaSequenceToStartTime.get(position); } - const firstSegmentUri = hlsSegments[0].absoluteUri; - shaka.log.debug('First segment', firstSegmentUri.split('/').pop(), - 'starts at', firstStartTime); - /** @type {!Array.} */ const references = []; + let previousReference = null; + for (let i = 0; i < hlsSegments.length; i++) { const item = hlsSegments[i]; - const previousReference = references[references.length - 1]; const startTime = (i == 0) ? firstStartTime : previousReference.endTime; position = mediaSequenceNumber + skippedSegments + i; @@ -2124,9 +2224,17 @@ shaka.hls.HlsParser = class { variables, playlist.absoluteUri, type); + previousReference = reference; if (reference) { - references.push(reference); + if (this.config_.hls.ignoreManifestProgramDateTime && + this.minSequenceNumber_ != null && + position < this.minSequenceNumber_) { + // This segment is ignored as part of our fallback synchronization + // method. + } else { + references.push(reference); + } } } @@ -2582,7 +2690,6 @@ shaka.hls.HlsParser = class { * stream: !shaka.extern.Stream, * verbatimMediaPlaylistUri: string, * absoluteMediaPlaylistUri: string, - * minTimestamp: number, * maxTimestamp: number, * mediaSequenceToStartTime: !Map., * canSkipSegments: boolean @@ -2601,8 +2708,6 @@ shaka.hls.HlsParser = class { * @property {string} absoluteMediaPlaylistUri * The absolute media playlist URI, resolved relative to the master playlist * and updated to reflect any redirects. - * @property {number} minTimestamp - * The minimum timestamp found in the stream. * @property {number} maxTimestamp * The maximum timestamp found in the stream. * @property {!Map.} mediaSequenceToStartTime diff --git a/lib/media/segment_index.js b/lib/media/segment_index.js index 6d03f3f768..b9fe48fb1e 100644 --- a/lib/media/segment_index.js +++ b/lib/media/segment_index.js @@ -94,6 +94,25 @@ shaka.media.SegmentIndex = class { } + /** + * Return the earliest reference, or null if empty. + * @return {shaka.media.SegmentReference} + */ + earliestReference() { + return this.references[0] || null; + } + + + /** + * Drop the first N references. + * Used in early HLS synchronization, and does not count as eviction. + * @param {number} n + */ + dropFirstReferences(n) { + this.references.splice(0, n); + } + + /** * Finds the position of the segment for the given time, in seconds, relative * to the start of the presentation. Returns the position of the segment @@ -166,7 +185,13 @@ shaka.media.SegmentIndex = class { for (const ref of this.references) { ref.startTime += offset; ref.endTime += offset; - ref.timestampOffset += offset; + ref.trueEndTime += offset; + + for (const partial of ref.partialReferences) { + partial.startTime += offset; + partial.endTime += offset; + partial.trueEndTime += offset; + } } } } diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index f6fe9f77c2..f504ab5237 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -160,8 +160,9 @@ shaka.media.SegmentReference = class { * If not provided, the duration should be automatically calculated based on * the duration of the reference. * @param {?number=} syncTime - * A time value, expressed in the same scale as the start and end time, which - * is used to synchronize between streams. + * A time value, expressed in seconds since 1970, which is used to + * synchronize between streams. Both produced and consumed by the HLS + * parser. Other components should not need this value. * @param {shaka.media.SegmentReference.Status=} status * The segment status is used to indicate that a segment does not exist or is * not available. @@ -221,7 +222,13 @@ shaka.media.SegmentReference = class { /** @type {?number} */ this.tileDuration = tileDuration; - /** @type {?number} */ + /** + * A time value, expressed in seconds since 1970, which is used to + * synchronize between streams. Both produced and consumed by the HLS + * parser. Other components should not need this value. + * + * @type {?number} + */ this.syncTime = syncTime; /** @type {shaka.media.SegmentReference.Status} */ diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index c5814758dd..fe9149233a 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1602,7 +1602,7 @@ shaka.media.StreamingEngine = class { await this.playerInterface_.mediaSourceEngine.appendBuffer( mediaState.type, segment, - reference.syncTime == null ? reference.startTime : reference.syncTime, + reference.startTime, reference.endTime, hasClosedCaptions, seeked); diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index dad19f5188..2c613823f2 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -2021,6 +2021,10 @@ describe('HlsParser', () => { }); describe('produces syncTime', () => { + // Corresponds to "2000-01-01T00:00:00.00Z". + // All the PROGRAM-DATE-TIME values in the tests below are at or after this. + const syncTimeBase = 946684800; + /** * @param {number} startTime * @param {number} endTime @@ -2039,9 +2043,10 @@ describe('HlsParser', () => { /** * @param {string} media * @param {!Array.} startTimes + * @param {number} syncTimeOffset * @param {(function(!shaka.media.SegmentReference))=} modifyFn */ - async function test(media, startTimes, modifyFn) { + async function test(media, startTimes, syncTimeOffset, modifyFn) { const master = [ '#EXTM3U\n', '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,vtt",', @@ -2053,7 +2058,8 @@ describe('HlsParser', () => { for (let i = 0; i < startTimes.length - 1; i++) { const startTime = startTimes[i]; const endTime = startTimes[i + 1]; - const reference = makeReference(startTime, endTime, startTime); + const reference = makeReference( + startTime, endTime, syncTimeOffset + startTime); if (modifyFn) { modifyFn(reference); } @@ -2100,7 +2106,7 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25]); + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5); }); it('when some EXT-X-PROGRAM-DATE-TIME values are missing', async () => { @@ -2122,7 +2128,7 @@ describe('HlsParser', () => { 'main.mp4\n', '#EXTINF:4,\n', 'main.mp4', - ].join(''), [0, 2, 7, 12, 17, 19, 23]); + ].join(''), [0, 2, 7, 12, 17, 19, 23], syncTimeBase + 8); }); it('except when ignoreManifestProgramDateTime is set', async () => { @@ -2148,7 +2154,7 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25], (reference) => { + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5, (reference) => { reference.syncTime = null; }); }); @@ -2176,11 +2182,11 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25], (reference) => { + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5, (reference) => { if (reference.startTime == 10) { reference.partialReferences = [ - makeReference(10, 12.5, 10), - makeReference(12.5, 15, 12.5), + makeReference(10, 12.5, syncTimeBase + 15), + makeReference(12.5, 15, syncTimeBase + 17.5), ]; } }); @@ -3555,4 +3561,92 @@ describe('HlsParser', () => { await testHlsParser(media, '', manifest); }); + + it('syncs on sequence with ignoreManifestProgramDateTime', async () => { + const config = shaka.util.PlayerConfiguration.createDefault().manifest; + config.hls.ignoreManifestProgramDateTime = true; + parser.configure(config); + + const master = [ + '#EXTM3U\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="eng",URI="audio"\n', + '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,mp4a",', + 'RESOLUTION=960x540,FRAME-RATE=60,AUDIO="aud1"\n', + 'video\n', + ].join(''); + + const video = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXT-X-MEDIA-SEQUENCE:1\n', + '#EXTINF:5,\n', + 'video1.mp4\n', + '#EXTINF:5,\n', + 'video2.mp4\n', + '#EXTINF:5,\n', + 'video3.mp4\n', + ].join(''); + + const audio = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXT-X-MEDIA-SEQUENCE:3\n', + '#EXTINF:5,\n', + 'audio3.mp4\n', + '#EXTINF:5,\n', + 'audio4.mp4\n', + ].join(''); + + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.sequenceMode = true; + manifest.anyTimeline(); + manifest.addPartialVariant((variant) => { + variant.language = 'en'; + variant.bandwidth = 200; + variant.addPartialStream(ContentType.VIDEO, (stream) => { + stream.mime('video/mp4', 'avc1'); + stream.size(960, 540); + stream.frameRate = 60; + }); + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.language = 'en'; + stream.mime('audio/mp4', 'mp4a'); + }); + }); + }); + + fakeNetEngine + .setResponseText('test:/master', master) + .setResponseText('test:/audio', audio) + .setResponseText('test:/video', video) + .setResponseValue('test:/init.mp4', initSegmentData); + + const actualManifest = await parser.start('test:/master', playerInterface); + expect(actualManifest).toEqual(manifest); + + const actualVideo = actualManifest.variants[0].video; + await actualVideo.createSegmentIndex(); + goog.asserts.assert(actualVideo.segmentIndex != null, 'Null segmentIndex!'); + + const actualAudio = actualManifest.variants[0].audio; + await actualAudio.createSegmentIndex(); + goog.asserts.assert(actualAudio.segmentIndex != null, 'Null segmentIndex!'); + + // Verify that the references are aligned on sequence number. + const videoSegments = Array.from(actualVideo.segmentIndex); + const audioSegments = Array.from(actualAudio.segmentIndex); + + // The first two were dropped to align with the audio. + expect(videoSegments.map((ref) => ref.getUris()[0])).toEqual([ + 'test:/video3.mp4', + ]); + // Audio has a 4th segment that video doesn't, but that doesn't get clipped + // or used as the base. Alignment is truly based on media sequence number. + expect(audioSegments.map((ref) => ref.getUris()[0])).toEqual([ + 'test:/audio3.mp4', + 'test:/audio4.mp4', + ]); + }); });