diff --git a/externs/shaka/player.js b/externs/shaka/player.js index 74576f01cc..f5bd8180cc 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -777,8 +777,10 @@ shaka.extern.DashManifestConfiguration; * Defaults to 'avc1.42E01E'. * @property {boolean} ignoreManifestProgramDateTime * If true, the HLS parser will ignore the - * EXT-X-PROGRAM-DATE-TIME tags in the manifest. - * Meant for tags that are incorrect or malformed. + * EXT-X-PROGRAM-DATE-TIME tags in the manifest and use media + * sequence numbers instead. + * Meant for streams where EXT-X-PROGRAM-DATE-TIME is incorrect + * or malformed. * Defaults to false. * @property {string} mediaPlaylistFullMimeType * A string containing a full mime type, including both the basic mime type diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index f59bbd8cfa..970012a3c1 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -121,13 +121,19 @@ shaka.hls.HlsParser = class { this.updatePlaylistDelay_ = 0; /** - * A time offset to apply to EXT-X-PROGRAM-DATE-TIME values to normalize - * them so that they start at 0. This is necessary because these times will - * be used to set presentation times for segments. - * null means we don't have enough data yet. + * If true, we have already calculated offsets to synchronize streams. + * Offsets are computed in syncStreams*_(). + * @private {boolean} + */ + this.streamsSynced_ = false; + + /** + * The minimum sequence number for generated segments, when ignoring + * EXT-X-PROGRAM-DATE-TIME. + * * @private {?number} */ - this.syncTimeOffset_ = null; + this.minSequenceNumber_ = null; /** * This timer is used to trigger the start of a manifest update. A manifest @@ -361,39 +367,155 @@ shaka.hls.HlsParser = class { } /** - * If necessary, makes sure that sync times will be normalized to 0, so that - * a stream does not start buffering at 50 years in because sync times are - * measured in time since 1970. + * Align all streams by sequence number by dropping early segments. Then + * offset all streams to begin at presentation time 0. * @private */ - calculateSyncTimeOffset_() { - if (this.syncTimeOffset_ != null) { - // The offset was already calculated. + syncStreamsWithSequenceNumber_() { + if (this.streamsSynced_) { return; } - const segments = new Set(); - let lowestSyncTime = Infinity; + // Sync using media sequence number. Find the highest starting sequence + // number among all streams. Later, we will drop any references to + // earlier segments in other streams, then offset everything back to 0. + let highestStartingSequenceNumber = -1; + const firstSequenceNumberMap = new Map(); + for (const streamInfo of this.uriToStreamInfosMap_.values()) { const segmentIndex = streamInfo.stream.segmentIndex; if (segmentIndex) { - segmentIndex.forEachTopLevelReference((segment) => { - if (segment.syncTime != null) { - lowestSyncTime = Math.min(lowestSyncTime, segment.syncTime); - segments.add(segment); + const segment0 = segmentIndex.earliestReference(); + if (segment0) { + // This looks inefficient, but iteration order is insertion order. + // So the very first entry should be the one we want. + // We assert that this holds true so that we are alerted by debug + // builds and tests if it changes. We still do a loop, though, so + // that the code functions correctly in production no matter what. + if (goog.DEBUG) { + const firstSequenceStartTime = + streamInfo.mediaSequenceToStartTime.values().next().value; + goog.asserts.assert( + firstSequenceStartTime == segment0.startTime, + 'Sequence number map is not ordered as expected!'); + } + for (const [sequence, start] of streamInfo.mediaSequenceToStartTime) { + if (start == segment0.startTime) { + firstSequenceNumberMap.set(streamInfo, sequence); + + highestStartingSequenceNumber = Math.max( + highestStartingSequenceNumber, sequence); + break; + } } - }); + } + } + } + + if (highestStartingSequenceNumber < 0) { + // Nothing to sync. + return; + } + + // From now on, updates will ignore any references before this number. + this.minSequenceNumber_ = highestStartingSequenceNumber; + + shaka.log.debug('Syncing HLS streams against base sequence number:', + this.minSequenceNumber_); + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex) { + // Drop any earlier references. + const numSegmentsToDrop = this.minSequenceNumber_ - + firstSequenceNumberMap.get(streamInfo); + segmentIndex.dropFirstReferences(numSegmentsToDrop); + + // Now adjust timestamps back to begin at 0. + const segmentN = segmentIndex.earliestReference(); + if (segmentN) { + this.offsetStream_(streamInfo, -segmentN.startTime); + } + } + } + + this.streamsSynced_ = true; + } + + /** + * Synchronize streams by the EXT-X-PROGRAM-DATE-TIME tags attached to their + * segments. Also normalizes segment times so that the earliest segment in + * any stream is at time 0. + * @private + */ + syncStreamsWithProgramDateTime_() { + if (this.streamsSynced_) { + return; + } + + let lowestSyncTime = Infinity; + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex) { + const segment0 = segmentIndex.earliestReference(); + if (segment0 != null && segment0.syncTime != null) { + lowestSyncTime = Math.min(lowestSyncTime, segment0.syncTime); + } } } - if (segments.size > 0) { - this.syncTimeOffset_ = -lowestSyncTime; - for (const segment of segments) { - segment.syncTime += this.syncTimeOffset_; - for (const partial of segment.partialReferences) { - partial.syncTime += this.syncTimeOffset_; + + if (lowestSyncTime == Infinity) { + // Nothing to sync. + return; + } + + shaka.log.debug('Syncing HLS streams against base time:', lowestSyncTime); + + for (const streamInfo of this.uriToStreamInfosMap_.values()) { + const segmentIndex = streamInfo.stream.segmentIndex; + if (segmentIndex != null) { + const segment0 = segmentIndex.earliestReference(); + if (segment0.syncTime == null) { + shaka.log.alwaysError('Missing EXT-X-PROGRAM-DATE-TIME for stream', + streamInfo.verbatimMediaPlaylistUri, + 'Expect AV sync issues!'); + } else { + // The first segment's target startTime should be based entirely on + // its syncTime. The rest of the stream will be based on that + // starting point. The earliest segment sync time from any stream + // will become presentation time 0. If two streams start e.g. 6 + // seconds apart in syncTime, then their first segments will also + // start 6 seconds apart in presentation time. + const segment0TargetTime = segment0.syncTime - lowestSyncTime; + const streamOffset = segment0TargetTime - segment0.startTime; + + this.offsetStream_(streamInfo, streamOffset); } } } + + this.streamsSynced_ = true; + } + + /** + * @param {!shaka.hls.HlsParser.StreamInfo} streamInfo + * @param {number} offset + * @private + */ + offsetStream_(streamInfo, offset) { + streamInfo.stream.segmentIndex.offset(offset); + + streamInfo.maxTimestamp += offset; + goog.asserts.assert(streamInfo.maxTimestamp >= 0, + 'Negative maxTimestamp after adjustment!'); + + for (const [key, value] of streamInfo.mediaSequenceToStartTime) { + streamInfo.mediaSequenceToStartTime.set(key, value + offset); + } + + shaka.log.debug('Offset', offset, 'applied to', + streamInfo.verbatimMediaPlaylistUri); } /** @@ -525,7 +647,11 @@ shaka.hls.HlsParser = class { // Now that we have generated all streams, we can determine the offset to // apply to sync times. - this.calculateSyncTimeOffset_(); + if (this.config_.hls.ignoreManifestProgramDateTime) { + this.syncStreamsWithSequenceNumber_(); + } else { + this.syncStreamsWithProgramDateTime_(); + } if (this.aesEncrypted_ && variants.length == 0) { // We do not support AES-128 encryption with HLS yet. Variants is null @@ -540,15 +666,12 @@ shaka.hls.HlsParser = class { // Find the min and max timestamp of the earliest segment in all streams. // Find the minimum duration of all streams as well. - let minFirstTimestamp = Infinity; let minDuration = Infinity; - for (const streamInfo of this.uriToStreamInfosMap_.values()) { - minFirstTimestamp = - Math.min(minFirstTimestamp, streamInfo.minTimestamp); if (streamInfo.stream.type != 'text') { - minDuration = Math.min(minDuration, - streamInfo.maxTimestamp - streamInfo.minTimestamp); + // Since everything is already offset to 0 (either by sync or by being + // VOD), only maxTimestamp is necessary to compute the duration. + minDuration = Math.min(minDuration, streamInfo.maxTimestamp); } } @@ -592,18 +715,11 @@ shaka.hls.HlsParser = class { segmentAvailabilityDuration); } } else { - // For VOD/EVENT content, offset everything back to 0. - // Use the minimum timestamp as the offset for all streams. // Use the minimum duration as the presentation duration. this.presentationTimeline_.setDuration(minDuration); - // Use a negative offset to adjust towards 0. - this.presentationTimeline_.offset(-minFirstTimestamp); for (const streamInfo of this.uriToStreamInfosMap_.values()) { - // The segments were created with actual media times, rather than - // presentation-aligned times, so offset them all now. - streamInfo.stream.segmentIndex.offset(-minFirstTimestamp); - // Finally, fit the segments to the playlist duration. + // Fit the segments to the playlist duration. streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration); } } @@ -811,12 +927,7 @@ shaka.hls.HlsParser = class { }); // Create stream info for each audio / video media tag. - // Wait for the first stream info created, so that the start time is fetched - // and can be reused. - if (mediaTags.length) { - await this.createStreamInfoFromMediaTag_(mediaTags[0]); - } - const promises = mediaTags.slice(1).map((tag) => { + const promises = mediaTags.map((tag) => { return this.createStreamInfoFromMediaTag_(tag); }); await Promise.all(promises); @@ -1584,7 +1695,6 @@ shaka.hls.HlsParser = class { throw error; } - const minTimestamp = segments[0].startTime; const lastEndTime = segments[segments.length - 1].endTime; /** @type {!shaka.media.SegmentIndex} */ const segmentIndex = new shaka.media.SegmentIndex(segments); @@ -1642,7 +1752,6 @@ shaka.hls.HlsParser = class { stream, verbatimMediaPlaylistUri, absoluteMediaPlaylistUri, - minTimestamp, maxTimestamp: lastEndTime, mediaSequenceToStartTime, canSkipSegments, @@ -1854,15 +1963,9 @@ shaka.hls.HlsParser = class { const dateTimeTag = shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-PROGRAM-DATE-TIME'); if (dateTimeTag && dateTimeTag.value) { - const time = shaka.util.XmlUtils.parseDate(dateTimeTag.value); - goog.asserts.assert(time != null, + syncTime = shaka.util.XmlUtils.parseDate(dateTimeTag.value); + goog.asserts.assert(syncTime != null, 'EXT-X-PROGRAM-DATE-TIME format not valid'); - // Sync time offset is null on the first go-through. This indicates that - // we have not yet seen every stream, and thus do not yet have enough - // information to determine how to normalize the sync times. - // For that first go-through, the sync time will be applied after the - // references are all created. Until then, just offset by 0. - syncTime = time + (this.syncTimeOffset_ || 0); } } @@ -2096,16 +2199,13 @@ shaka.hls.HlsParser = class { firstStartTime = mediaSequenceToStartTime.get(position); } - const firstSegmentUri = hlsSegments[0].absoluteUri; - shaka.log.debug('First segment', firstSegmentUri.split('/').pop(), - 'starts at', firstStartTime); - /** @type {!Array.} */ const references = []; + let previousReference = null; + for (let i = 0; i < hlsSegments.length; i++) { const item = hlsSegments[i]; - const previousReference = references[references.length - 1]; const startTime = (i == 0) ? firstStartTime : previousReference.endTime; position = mediaSequenceNumber + skippedSegments + i; @@ -2135,9 +2235,17 @@ shaka.hls.HlsParser = class { variables, playlist.absoluteUri, type); + previousReference = reference; if (reference) { - references.push(reference); + if (this.config_.hls.ignoreManifestProgramDateTime && + this.minSequenceNumber_ != null && + position < this.minSequenceNumber_) { + // This segment is ignored as part of our fallback synchronization + // method. + } else { + references.push(reference); + } } } @@ -2593,7 +2701,6 @@ shaka.hls.HlsParser = class { * stream: !shaka.extern.Stream, * verbatimMediaPlaylistUri: string, * absoluteMediaPlaylistUri: string, - * minTimestamp: number, * maxTimestamp: number, * mediaSequenceToStartTime: !Map., * canSkipSegments: boolean @@ -2612,8 +2719,6 @@ shaka.hls.HlsParser = class { * @property {string} absoluteMediaPlaylistUri * The absolute media playlist URI, resolved relative to the master playlist * and updated to reflect any redirects. - * @property {number} minTimestamp - * The minimum timestamp found in the stream. * @property {number} maxTimestamp * The maximum timestamp found in the stream. * @property {!Map.} mediaSequenceToStartTime diff --git a/lib/media/segment_index.js b/lib/media/segment_index.js index 6d03f3f768..b9fe48fb1e 100644 --- a/lib/media/segment_index.js +++ b/lib/media/segment_index.js @@ -94,6 +94,25 @@ shaka.media.SegmentIndex = class { } + /** + * Return the earliest reference, or null if empty. + * @return {shaka.media.SegmentReference} + */ + earliestReference() { + return this.references[0] || null; + } + + + /** + * Drop the first N references. + * Used in early HLS synchronization, and does not count as eviction. + * @param {number} n + */ + dropFirstReferences(n) { + this.references.splice(0, n); + } + + /** * Finds the position of the segment for the given time, in seconds, relative * to the start of the presentation. Returns the position of the segment @@ -166,7 +185,13 @@ shaka.media.SegmentIndex = class { for (const ref of this.references) { ref.startTime += offset; ref.endTime += offset; - ref.timestampOffset += offset; + ref.trueEndTime += offset; + + for (const partial of ref.partialReferences) { + partial.startTime += offset; + partial.endTime += offset; + partial.trueEndTime += offset; + } } } } diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index f6fe9f77c2..f504ab5237 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -160,8 +160,9 @@ shaka.media.SegmentReference = class { * If not provided, the duration should be automatically calculated based on * the duration of the reference. * @param {?number=} syncTime - * A time value, expressed in the same scale as the start and end time, which - * is used to synchronize between streams. + * A time value, expressed in seconds since 1970, which is used to + * synchronize between streams. Both produced and consumed by the HLS + * parser. Other components should not need this value. * @param {shaka.media.SegmentReference.Status=} status * The segment status is used to indicate that a segment does not exist or is * not available. @@ -221,7 +222,13 @@ shaka.media.SegmentReference = class { /** @type {?number} */ this.tileDuration = tileDuration; - /** @type {?number} */ + /** + * A time value, expressed in seconds since 1970, which is used to + * synchronize between streams. Both produced and consumed by the HLS + * parser. Other components should not need this value. + * + * @type {?number} + */ this.syncTime = syncTime; /** @type {shaka.media.SegmentReference.Status} */ diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index 4e1bee95ea..2882b2d59f 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1609,7 +1609,7 @@ shaka.media.StreamingEngine = class { await this.playerInterface_.mediaSourceEngine.appendBuffer( mediaState.type, segment, - reference.syncTime == null ? reference.startTime : reference.syncTime, + reference.startTime, reference.endTime, hasClosedCaptions, seeked); diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index dad19f5188..2c613823f2 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -2021,6 +2021,10 @@ describe('HlsParser', () => { }); describe('produces syncTime', () => { + // Corresponds to "2000-01-01T00:00:00.00Z". + // All the PROGRAM-DATE-TIME values in the tests below are at or after this. + const syncTimeBase = 946684800; + /** * @param {number} startTime * @param {number} endTime @@ -2039,9 +2043,10 @@ describe('HlsParser', () => { /** * @param {string} media * @param {!Array.} startTimes + * @param {number} syncTimeOffset * @param {(function(!shaka.media.SegmentReference))=} modifyFn */ - async function test(media, startTimes, modifyFn) { + async function test(media, startTimes, syncTimeOffset, modifyFn) { const master = [ '#EXTM3U\n', '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,vtt",', @@ -2053,7 +2058,8 @@ describe('HlsParser', () => { for (let i = 0; i < startTimes.length - 1; i++) { const startTime = startTimes[i]; const endTime = startTimes[i + 1]; - const reference = makeReference(startTime, endTime, startTime); + const reference = makeReference( + startTime, endTime, syncTimeOffset + startTime); if (modifyFn) { modifyFn(reference); } @@ -2100,7 +2106,7 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25]); + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5); }); it('when some EXT-X-PROGRAM-DATE-TIME values are missing', async () => { @@ -2122,7 +2128,7 @@ describe('HlsParser', () => { 'main.mp4\n', '#EXTINF:4,\n', 'main.mp4', - ].join(''), [0, 2, 7, 12, 17, 19, 23]); + ].join(''), [0, 2, 7, 12, 17, 19, 23], syncTimeBase + 8); }); it('except when ignoreManifestProgramDateTime is set', async () => { @@ -2148,7 +2154,7 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25], (reference) => { + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5, (reference) => { reference.syncTime = null; }); }); @@ -2176,11 +2182,11 @@ describe('HlsParser', () => { '#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:25.00Z\n', '#EXTINF:5,\n', 'main.mp4', - ].join(''), [0, 5, 10, 15, 20, 25], (reference) => { + ].join(''), [0, 5, 10, 15, 20, 25], syncTimeBase + 5, (reference) => { if (reference.startTime == 10) { reference.partialReferences = [ - makeReference(10, 12.5, 10), - makeReference(12.5, 15, 12.5), + makeReference(10, 12.5, syncTimeBase + 15), + makeReference(12.5, 15, syncTimeBase + 17.5), ]; } }); @@ -3555,4 +3561,92 @@ describe('HlsParser', () => { await testHlsParser(media, '', manifest); }); + + it('syncs on sequence with ignoreManifestProgramDateTime', async () => { + const config = shaka.util.PlayerConfiguration.createDefault().manifest; + config.hls.ignoreManifestProgramDateTime = true; + parser.configure(config); + + const master = [ + '#EXTM3U\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="eng",URI="audio"\n', + '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,mp4a",', + 'RESOLUTION=960x540,FRAME-RATE=60,AUDIO="aud1"\n', + 'video\n', + ].join(''); + + const video = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXT-X-MEDIA-SEQUENCE:1\n', + '#EXTINF:5,\n', + 'video1.mp4\n', + '#EXTINF:5,\n', + 'video2.mp4\n', + '#EXTINF:5,\n', + 'video3.mp4\n', + ].join(''); + + const audio = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXT-X-MEDIA-SEQUENCE:3\n', + '#EXTINF:5,\n', + 'audio3.mp4\n', + '#EXTINF:5,\n', + 'audio4.mp4\n', + ].join(''); + + const manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.sequenceMode = true; + manifest.anyTimeline(); + manifest.addPartialVariant((variant) => { + variant.language = 'en'; + variant.bandwidth = 200; + variant.addPartialStream(ContentType.VIDEO, (stream) => { + stream.mime('video/mp4', 'avc1'); + stream.size(960, 540); + stream.frameRate = 60; + }); + variant.addPartialStream(ContentType.AUDIO, (stream) => { + stream.language = 'en'; + stream.mime('audio/mp4', 'mp4a'); + }); + }); + }); + + fakeNetEngine + .setResponseText('test:/master', master) + .setResponseText('test:/audio', audio) + .setResponseText('test:/video', video) + .setResponseValue('test:/init.mp4', initSegmentData); + + const actualManifest = await parser.start('test:/master', playerInterface); + expect(actualManifest).toEqual(manifest); + + const actualVideo = actualManifest.variants[0].video; + await actualVideo.createSegmentIndex(); + goog.asserts.assert(actualVideo.segmentIndex != null, 'Null segmentIndex!'); + + const actualAudio = actualManifest.variants[0].audio; + await actualAudio.createSegmentIndex(); + goog.asserts.assert(actualAudio.segmentIndex != null, 'Null segmentIndex!'); + + // Verify that the references are aligned on sequence number. + const videoSegments = Array.from(actualVideo.segmentIndex); + const audioSegments = Array.from(actualAudio.segmentIndex); + + // The first two were dropped to align with the audio. + expect(videoSegments.map((ref) => ref.getUris()[0])).toEqual([ + 'test:/video3.mp4', + ]); + // Audio has a 4th segment that video doesn't, but that doesn't get clipped + // or used as the base. Alignment is truly based on media sequence number. + expect(audioSegments.map((ref) => ref.getUris()[0])).toEqual([ + 'test:/audio3.mp4', + 'test:/audio4.mp4', + ]); + }); });