From b16d30c249586ae569fc9d632e3be593cb095ed8 Mon Sep 17 00:00:00 2001 From: Michelle Zhuo Date: Fri, 25 Jun 2021 07:30:12 -0700 Subject: [PATCH] feat(dash): Create segment index only when used Only create the segment index for a stream when the stream is chosen. When switching to another stream, release the segment index of the original stream, and create the segment index of the new stream during the next update. Change-Id: I4d8a64e0e52d3e7edb71d402a97ab1dcd7f561dd --- externs/shaka/manifest.js | 13 ++- lib/media/streaming_engine.js | 13 ++- lib/util/periods.js | 162 +++++++++++++++++------------ test/dash/dash_parser_live_unit.js | 100 +++++++++++++++++- 4 files changed, 221 insertions(+), 67 deletions(-) diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js index 968e8fafb3..a4f055fed8 100644 --- a/externs/shaka/manifest.js +++ b/externs/shaka/manifest.js @@ -238,6 +238,7 @@ shaka.extern.CreateSegmentIndexFunction; * id: number, * originalId: ?string, * createSegmentIndex: shaka.extern.CreateSegmentIndexFunction, + * closeSegmentIndex: (function()|undefined), * segmentIndex: shaka.media.SegmentIndex, * mimeType: string, * codecs: string, @@ -263,7 +264,10 @@ shaka.extern.CreateSegmentIndexFunction; * audioSamplingRate: ?number, * spatialAudio: boolean, * closedCaptions: Map., - * tilesLayout: (string|undefined) + * tilesLayout: (string|undefined), + * matchedStreams: + * (!Array.|!Array.| + * undefined) * }} * * @description @@ -280,6 +284,9 @@ shaka.extern.CreateSegmentIndexFunction; * @property {shaka.extern.CreateSegmentIndexFunction} createSegmentIndex * Required.
* Creates the Stream's segmentIndex (asynchronously). + * @property {(function()|undefined)} closeSegmentIndex + * Optional.
+ * Closes the Stream's segmentIndex. * @property {shaka.media.SegmentIndex} segmentIndex * Required.
* May be null until createSegmentIndex() is complete. @@ -370,6 +377,10 @@ shaka.extern.CreateSegmentIndexFunction; * The value is a grid-item-dimension consisting of two positive decimal * integers in the format: column-x-row ('4x3'). It describes the arrangement * of Images in a Grid. The minimum valid LAYOUT is '1x1'. + * @property {(!Array.|!Array.| + * undefined)} matchedStreams + * The streams in all periods which match the stream. Used for Dash. + * * @exportDoc */ shaka.extern.Stream; diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index d62d1511e1..c32a3426db 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -431,6 +431,11 @@ shaka.media.StreamingEngine = class { this.playerInterface_.mediaSourceEngine.reinitText(fullMimeType); } + // Releases the segmentIndex of the old stream. + if (mediaState.stream.closeSegmentIndex) { + mediaState.stream.closeSegmentIndex(); + } + mediaState.stream = stream; mediaState.segmentIterator = null; @@ -863,7 +868,7 @@ shaka.media.StreamingEngine = class { return; } - // Make sure the segment index exists. + // Make sure the segment index exists. If not, create the segment index. if (!mediaState.stream.segmentIndex) { const thisStream = mediaState.stream; @@ -873,6 +878,12 @@ shaka.media.StreamingEngine = class { // We switched streams while in the middle of this async call to // createSegmentIndex. Abandon this update and schedule a new one if // there's not already one pending. + // Releases the segmentIndex of the old stream. + if (thisStream.closeSegmentIndex) { + goog.asserts.assert(!mediaState.stream.segmentIndex, + 'mediastate.stream should not have segmentIndex yet.'); + thisStream.closeSegmentIndex(); + } if (mediaState.updateTimer == null) { this.scheduleUpdate_(mediaState, 0); } diff --git a/lib/util/periods.js b/lib/util/periods.js index 4f2ca3a496..ec15872c6a 100644 --- a/lib/util/periods.js +++ b/lib/util/periods.js @@ -544,8 +544,7 @@ shaka.util.PeriodCombiner = class { for (const stream of unusedStreams) { // Create a new output stream which includes this input stream. const outputStream = - // eslint-disable-next-line no-await-in-loop - await shaka.util.PeriodCombiner.createNewOutputStream_( + shaka.util.PeriodCombiner.createNewOutputStream_( stream, streamsPerPeriod, clone, concat, unusedStreamsPerPeriod); if (outputStream) { @@ -616,52 +615,69 @@ shaka.util.PeriodCombiner = class { static async extendExistingOutputStream_( outputStream, streamsPerPeriod, firstNewPeriodIndex, concat, unusedStreamsPerPeriod) { - const matches = shaka.util.PeriodCombiner.findMatchesInAllPeriods_( - streamsPerPeriod, outputStream); - - if (!matches) { - // We were unable to extend this output stream. - shaka.log.error('No matches extending output stream!', - outputStream, streamsPerPeriod); - return false; - } + shaka.util.PeriodCombiner.findMatchesInAllPeriods_(streamsPerPeriod, + outputStream); // This only exists where T == Stream, and this should only ever be called // on Stream types. StreamDB should not have pre-existing output streams. goog.asserts.assert(outputStream.createSegmentIndex, 'outputStream should be a Stream type!'); + if (!outputStream.matchedStreams) { + // We were unable to extend this output stream. + shaka.log.error('No matches extending output stream!', + outputStream, streamsPerPeriod); + return false; + } // We need to create all the per-period segment indexes and append them to // the output's MetaSegmentIndex. - await shaka.util.PeriodCombiner.createSegmentIndexes_(matches); - - // Assure the compiler that matches didn't become null during the async - // operation above. - goog.asserts.assert(matches, 'Matches should be non-null'); + if (outputStream.segmentIndex) { + await shaka.util.PeriodCombiner.extendOutputSegmentIndex_(outputStream, + firstNewPeriodIndex); + } - shaka.util.PeriodCombiner.extendOutputStream_( - outputStream, matches, firstNewPeriodIndex, concat, - unusedStreamsPerPeriod); + shaka.util.PeriodCombiner.extendOutputStream_(outputStream, + firstNewPeriodIndex, concat, unusedStreamsPerPeriod); return true; } /** - * Creates all segment indexes for an array of streams. Returns once every - * segment index is created. + * Creates the segment indexes for an array of input streams, and append them + * to the output stream's segment index. * - * @param {!Array.} streams - * @return {!Promise} + * @param {shaka.extern.Stream} outputStream + * @param {number} firstNewPeriodIndex An index into streamsPerPeriod which + * represents the first new period that hasn't been processed yet. * @private */ - static createSegmentIndexes_(streams) { + static async extendOutputSegmentIndex_(outputStream, firstNewPeriodIndex) { const operations = []; + const streams = outputStream.matchedStreams; + goog.asserts.assert(streams, 'matched streams should be valid'); + for (const stream of streams) { operations.push(stream.createSegmentIndex()); if (stream.trickModeVideo && !stream.trickModeVideo.segmentIndex) { operations.push(stream.trickModeVideo.createSegmentIndex()); } } - return Promise.all(operations); + await Promise.all(operations); + + // Concatenate the new matches onto the stream, starting at the first new + // period. + const Iterables = shaka.util.Iterables; + // Satisfy the compiler about the type. + // Also checks if the segmentIndex is still valid after the async + // operations, to make sure we stop if the active stream has changed. + if (outputStream.segmentIndex instanceof shaka.media.MetaSegmentIndex) { + for (const {i, item: match} of Iterables.enumerate(streams)) { + if (match.segmentIndex && i >= firstNewPeriodIndex) { + goog.asserts.assert(match.segmentIndex, + 'stream should have a segmentIndex.'); + outputStream.segmentIndex.appendSegmentIndex(match.segmentIndex); + } + } + } } /** @@ -679,7 +695,7 @@ shaka.util.PeriodCombiner = class { * @param {!Array.>} unusedStreamsPerPeriod An array of sets of * unused streams from each period. * - * @return {!Promise.} A newly-created output Stream, or null if matches + * @return {?T} A newly-created output Stream, or null if matches * could not be found.` * * @template T @@ -687,35 +703,36 @@ shaka.util.PeriodCombiner = class { * * @private */ - static async createNewOutputStream_( + static createNewOutputStream_( stream, streamsPerPeriod, clone, concat, unusedStreamsPerPeriod) { // Start by cloning the stream without segments, key IDs, etc. const outputStream = clone(stream); // Find best-matching streams in all periods. - const matches = shaka.util.PeriodCombiner.findMatchesInAllPeriods_( - streamsPerPeriod, outputStream); - - if (!matches) { - // This is not a stream we can build output from, but it may become part - // of another output based on another period's stream. - return null; - } + shaka.util.PeriodCombiner.findMatchesInAllPeriods_(streamsPerPeriod, + outputStream); // This only exists where T == Stream. if (outputStream.createSegmentIndex) { + // Override the createSegmentIndex function of the outputStream. + outputStream.createSegmentIndex = async () => { + if (!outputStream.segmentIndex) { + outputStream.segmentIndex = new shaka.media.MetaSegmentIndex(); + await shaka.util.PeriodCombiner.extendOutputSegmentIndex_( + outputStream, /* firstNewPeriodIndex= */ 0); + } + }; // For T == Stream, we need to create all the per-period segment indexes // in advance. concat() will add them to the output's MetaSegmentIndex. - await shaka.util.PeriodCombiner.createSegmentIndexes_(matches); } - // Assure the compiler that matches didn't become null during the async - // operation above. - goog.asserts.assert(matches, 'Matches should be non-null'); - - shaka.util.PeriodCombiner.extendOutputStream_( - outputStream, matches, /* firstNewPeriodIndex= */ 0, concat, - unusedStreamsPerPeriod); + if (!outputStream.matchedStreams) { + // This is not a stream we can build output from, but it may become part + // of another output based on another period's stream. + return null; + } + shaka.util.PeriodCombiner.extendOutputStream_(outputStream, + /* firstNewPeriodIndex= */ 0, concat, unusedStreamsPerPeriod); return outputStream; } @@ -723,7 +740,6 @@ shaka.util.PeriodCombiner = class { /** * @param {T} outputStream An existing output stream which needs to be * extended into new periods. - * @param {!Array.} matches A list of matching Streams from each period. * @param {number} firstNewPeriodIndex An index into streamsPerPeriod which * represents the first new period that hasn't been processed yet. * @param {function(T, T)} concat Concatenate the second stream onto the end @@ -737,10 +753,15 @@ shaka.util.PeriodCombiner = class { * @private */ static extendOutputStream_( - outputStream, matches, firstNewPeriodIndex, concat, - unusedStreamsPerPeriod) { + outputStream, firstNewPeriodIndex, concat, unusedStreamsPerPeriod) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const LanguageUtils = shaka.util.LanguageUtils; + const matches = outputStream.matchedStreams; + + // Assure the compiler that matches didn't become null during the async + // operation before. + goog.asserts.assert(outputStream.matchedStreams, + 'matchedStreams should be non-null'); // Concatenate the new matches onto the stream, starting at the first new // period. @@ -784,7 +805,23 @@ shaka.util.PeriodCombiner = class { // streams that match this output. clone.originalId = null; clone.createSegmentIndex = () => Promise.resolve(); - clone.segmentIndex = new shaka.media.MetaSegmentIndex(); + clone.closeSegmentIndex = () => { + if (clone.segmentIndex) { + clone.segmentIndex.release(); + clone.segmentIndex = null; + } + // Close the segment index of the matched streams. + if (clone.matchedStreams) { + for (const match of clone.matchedStreams) { + if (match.segmentIndex) { + match.segmentIndex.release(); + match.segmentIndex = null; + } + } + } + }; + + clone.segmentIndex = null; clone.emsgSchemeIdUris = []; clone.keyIds = new Set(); clone.closedCaptions = null; @@ -870,26 +907,24 @@ shaka.util.PeriodCombiner = class { } } - // Satisfy the compiler about the type. - goog.asserts.assert( - output.segmentIndex instanceof shaka.media.MetaSegmentIndex, - 'Output streams should have a MetaSegmentIndex!'); - // Satisfy the compiler that the input index has been created. - goog.asserts.assert( - input.segmentIndex, - 'Input segment index should have been created by now!'); - - output.segmentIndex.appendSegmentIndex(input.segmentIndex); - // Combine trick-play video streams, if present. if (input.trickModeVideo) { if (!output.trickModeVideo) { // Create a fresh output stream for trick-mode playback. output.trickModeVideo = shaka.util.PeriodCombiner.cloneStream_( input.trickModeVideo); - // Start it with whatever non-trick-mode Streams are in the output so - // far. - output.trickModeVideo.segmentIndex = output.segmentIndex.clone(); + // TODO: fix the createSegmentIndex function for trickModeVideo. + // The trick-mode tracks in multi-period content should have trick-mode + // segment indexes whenever available, rather than only regular-mode + // segment indexes. + output.trickModeVideo.createSegmentIndex = () => { + // Satisfy the compiler about the type. + goog.asserts.assert( + output.segmentIndex instanceof shaka.media.MetaSegmentIndex, + 'The stream should have a MetaSegmentIndex.'); + output.trickModeVideo.segmentIndex = output.segmentIndex.clone(); + return Promise.resolve(); + }; } // Concatenate the trick mode input onto the trick mode output. @@ -940,7 +975,6 @@ shaka.util.PeriodCombiner = class { * * @param {!Array.>} streamsPerPeriod * @param {T} outputStream - * @return {Array.} * * @template T * Accepts either a StreamDB or Stream type. @@ -953,11 +987,11 @@ shaka.util.PeriodCombiner = class { const match = shaka.util.PeriodCombiner.findBestMatchInPeriod_( streams, outputStream); if (!match) { - return null; + return; } matches.push(match); } - return matches; + outputStream.matchedStreams = matches; } /** diff --git a/test/dash/dash_parser_live_unit.js b/test/dash/dash_parser_live_unit.js index 5657f177ec..25806a7723 100644 --- a/test/dash/dash_parser_live_unit.js +++ b/test/dash/dash_parser_live_unit.js @@ -364,7 +364,7 @@ describe('DashParser Live', () => { }); } - it('can add Periods', async () => { + it('can add Periods with SegmentTemplate', async () => { const template1 = [ ' { // two segments (10-14s). expect(stream.segmentIndex.find(9)).not.toBe(null); expect(stream.segmentIndex.find(13)).not.toBe(null); + + stream.closeSegmentIndex(); + await stream.createSegmentIndex(); + + expect(stream.segmentIndex.find(9)).not.toBe(null); + expect(stream.segmentIndex.find(13)).not.toBe(null); + }); + + it('can add Periods with SegmentList', async () => { + const list1 = [ + '', + ' ', + ' ', + ' ', + ' http://example.com', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'); + const list2 = [ + '', + ' ', + ' ', + ' ', + ' http://example.com', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' http://example.com', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'); + const firstManifest = sprintf(list1, {updateTime: updateTime}); + const secondManifest = sprintf(list2, {updateTime: updateTime}); + + fakeNetEngine.setResponseText('dummy://foo', firstManifest); + // First three segments should exist. + Date.now = () => 5; + + const manifest = await parser.start('dummy://foo', playerInterface); + const variant = manifest.variants[0]; + const stream = variant.video; + await stream.createSegmentIndex(); + + // First three segments exist, but not the fourth. + expect(stream.segmentIndex.find(25)).not.toBe(null); + expect(stream.segmentIndex.find(45)).toBe(null); + + fakeNetEngine.setResponseText('dummy://foo', secondManifest); + Date.now = () => 25; + + await updateManifest(); + + // The update should have affected the same variant object we captured + // before. Now the entire first period should exist (0-40s), plus the next + // two segments of the second period(40-60s). + expect(stream.segmentIndex.find(25)).not.toBe(null); + expect(stream.segmentIndex.find(45)).not.toBe(null); + + stream.closeSegmentIndex(); + await stream.createSegmentIndex(); + + expect(stream.segmentIndex.find(25)).not.toBe(null); + expect(stream.segmentIndex.find(45)).not.toBe(null); }); it('uses redirect URL for manifest BaseURL and updates', async () => {