Skip to content

Commit

Permalink
feat: Improve sequence mode start time (shaka-project#5326)
Browse files Browse the repository at this point in the history
  • Loading branch information
@avelad
avelad committed Jun 20, 2023
1 parent ea896d5 commit 80cacf6
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 141 deletions.
97 changes: 13 additions & 84 deletions lib/media/media_source_engine.js
View file
Expand Up @@ -133,9 +133,6 @@ shaka.media.MediaSourceEngine = class {
/** @private {boolean} */
this.ignoreManifestTimestampsInSegmentsMode_ = false;

/** @private {boolean} */
this.hasTextStreams_ = true;

/** @private {!shaka.util.PublicPromise.<number>} */
this.textSequenceModeOffset_ = new shaka.util.PublicPromise();
}
Expand Down Expand Up @@ -372,22 +369,19 @@ shaka.media.MediaSourceEngine = class {
* segment durations being out of sync with segment durations. In other
* words, assume that there are no gaps in the segments when appending
* to the SourceBuffer, even if the manifest and segment times disagree.
* @param {boolean=} hasTextStreams
* Indicates if the manifest has text streams.
*
* @return {!Promise}
*/
async init(streamsByType, sequenceMode=false,
manifestType=shaka.media.ManifestParser.UNKNOWN,
ignoreManifestTimestampsInSegmentsMode=false,
hasTextStreams=true) {
ignoreManifestTimestampsInSegmentsMode=false) {
await this.mediaSourceOpen_;

this.sequenceMode_ = sequenceMode;
this.manifestType_ = manifestType;
this.ignoreManifestTimestampsInSegmentsMode_ =
ignoreManifestTimestampsInSegmentsMode;
this.hasTextStreams_ = hasTextStreams;

for (const contentType of streamsByType.keys()) {
const stream = streamsByType.get(contentType);
Expand Down Expand Up @@ -454,9 +448,7 @@ shaka.media.MediaSourceEngine = class {
'expected \'open\'');
}

if (this.sequenceMode_ && !this.hasTextStreams_) {
// There's no text streams, so we can set sequence mode early instead
// of setting it after the first segment is appended in appendBuffer_.
if (this.sequenceMode_) {
sourceBuffer.mode =
shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE;
}
Expand Down Expand Up @@ -652,7 +644,7 @@ shaka.media.MediaSourceEngine = class {
* we are appending, or null for init segments
* @param {!string} mimeType
* @param {!number} timestampOffset
* @return {number}
* @return {?number}
* @private
*/
getTimestampAndDispatchMetadata_(contentType, data, reference, mimeType,
Expand Down Expand Up @@ -771,7 +763,7 @@ shaka.media.MediaSourceEngine = class {
}
const timestamp = this.getTimestampAndDispatchMetadata_(
contentType, data, reference, mimeType, timestampOffset);
if (attemptTimestampOffsetCalculation) {
if (timestamp != null && reference) {
const calculatedTimestampOffset = reference.startTime - timestamp;
const timestampOffsetDifference =
Math.abs(timestampOffset - calculatedTimestampOffset);
Expand All @@ -784,6 +776,15 @@ shaka.media.MediaSourceEngine = class {
() => this.setTimestampOffset_(contentType, timestampOffset));
}
}
// Timestamps can only be reliably extracted from video, not audio.
// Packed audio formats do not have internal timestamps at all.
// Prefer video for this when available.
const isBestSourceBufferForTimestamps =
contentType == ContentType.VIDEO ||
!(ContentType.VIDEO in this.sourceBuffers_);
if (this.sequenceMode_ && isBestSourceBufferForTimestamps) {
this.textSequenceModeOffset_.resolve(timestampOffset);
}
}
if (hasClosedCaptions && contentType == ContentType.VIDEO) {
if (!this.textEngine_) {
Expand Down Expand Up @@ -818,78 +819,6 @@ shaka.media.MediaSourceEngine = class {
data = this.workAroundBrokenPlatforms_(
data, reference ? reference.startTime : null, contentType);

const sourceBuffer = this.sourceBuffers_[contentType];
const SEQUENCE = shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE;

if (this.sequenceMode_ && sourceBuffer.mode != SEQUENCE && reference) {
// This is the first media segment to be appended to a SourceBuffer in
// sequence mode. We set the mode late so that we can trick MediaSource
// into extracting a timestamp for us to align text segments in sequence
// mode.

const duration = this.mediaSource_.duration;

// Timestamps can only be reliably extracted from video, not audio.
// Packed audio formats do not have internal timestamps at all.
// Prefer video for this when available.
const isBestSourceBufferForTimestamps =
contentType == ContentType.VIDEO ||
!(ContentType.VIDEO in this.sourceBuffers_);
if (isBestSourceBufferForTimestamps) {
// Append the segment in segments mode first, with offset of 0 and an
// open append window.
const originalRange =
[sourceBuffer.appendWindowStart, sourceBuffer.appendWindowEnd];
sourceBuffer.appendWindowStart = 0;
sourceBuffer.appendWindowEnd = Infinity;

const originalOffset = sourceBuffer.timestampOffset;
sourceBuffer.timestampOffset = 0;

await this.enqueueOperation_(
contentType, () => this.append_(contentType, data));
// If the input buffer passed to SourceBuffer#appendBuffer() does not
// contain a complete media segment, the call will exit while the
// SourceBuffer's append state is
// still PARSING_MEDIA_SEGMENT. Reset the parser state by calling
// abort() to safely reset timestampOffset to 'originalOffset'.
// https://www.w3.org/TR/media-source-2/#sourcebuffer-segment-parser-loop
await this.enqueueOperation_(
contentType, () => this.abort_(contentType));

// Reset the offset and append window.
sourceBuffer.timestampOffset = originalOffset;
sourceBuffer.appendWindowStart = originalRange[0];
sourceBuffer.appendWindowEnd = originalRange[1];

// Now get the timestamp of the segment and compute the offset for text
// segments.
const mediaStartTime = shaka.media.TimeRangesUtils.bufferStart(
this.getBuffered_(contentType));
const textOffset = (reference.startTime || 0) - (mediaStartTime || 0);
this.textSequenceModeOffset_.resolve(textOffset);

// Clear the buffer.
await this.enqueueOperation_(
contentType,
() => this.remove_(contentType, 0, duration));

// Finally, flush the buffer in case of choppy video start on HLS fMP4.
if (contentType == ContentType.VIDEO) {
await this.enqueueOperation_(
contentType,
() => this.flush_(contentType));
}
}

// Now switch to sequence mode and fall through to our normal operations.
sourceBuffer.mode = SEQUENCE;

// When we change the buffer mode the duration is lost, so we need to set
// it explicitly.
await this.setDuration(duration);
}

if (reference && this.sequenceMode_ && contentType != ContentType.TEXT) {
// In sequence mode, for non-text streams, if we just cleared the buffer
// and are either performing an unbuffered seek or handling an automatic
Expand Down
1 change: 0 additions & 1 deletion lib/media/streaming_engine.js
View file
Expand Up @@ -831,7 +831,6 @@ shaka.media.StreamingEngine = class {
this.manifest_.sequenceMode,
this.manifest_.type,
this.manifest_.ignoreManifestTimestampsInSegmentsMode,
this.manifest_.textStreams.length > 0,
);
this.destroyer_.ensureNotDestroyed();

Expand Down
54 changes: 0 additions & 54 deletions test/media/media_source_engine_unit.js
View file
Expand Up @@ -667,60 +667,6 @@ describe('MediaSourceEngine', () => {

expect(videoSourceBuffer.timestampOffset).toBe(0.50);
});

it('calls abort before setting timestampOffset', async () => {
const simulateUpdate = async () => {
await Util.shortDelay();
videoSourceBuffer.updateend();
};
const initObject = new Map();
initObject.set(ContentType.VIDEO, fakeVideoStream);

await mediaSourceEngine.init(initObject, /* sequenceMode= */ true);

// First, mock the scenario where timestampOffset is set to help align
// text segments. In this case, SourceBuffer mode is still 'segments'.
let reference = dummyReference(0, 1000);
let appendVideo = mediaSourceEngine.appendBuffer(
ContentType.VIDEO, buffer, reference, fakeStream,
/* hasClosedCaptions= */ false);
// Wait for the first appendBuffer(), in segments mode.
await simulateUpdate();
// Next, wait for abort(), used to reset the parser state for a safe
// setting of timestampOffset. Shaka fakes an updateend event on abort(),
// so simulateUpdate() isn't needed.
await Util.shortDelay();
// Next, wait for remove(), used to clear the SourceBuffer from the
// initial append.
await simulateUpdate();
// Next, wait for the second appendBuffer(), falling through to normal
// operations.
await simulateUpdate();
// Lastly, wait for the function-scoped MediaSourceEngine#appendBuffer()
// promise to resolve.
await appendVideo;
expect(videoSourceBuffer.abort).toHaveBeenCalledTimes(1);

// Second, mock the scenario where timestampOffset is set during an
// unbuffered seek or adaptation. SourceBuffer mode is 'sequence' now.
reference = dummyReference(0, 1000);
appendVideo = mediaSourceEngine.appendBuffer(
ContentType.VIDEO, buffer, reference, fakeStream,
/* hasClosedCaptions= */ false, /* seeked= */ true);
// First, wait for abort(), used to reset the parser state for a safe
// setting of timestampOffset.
await Util.shortDelay();
// The subsequent setTimestampOffset() fakes an updateend event for us, so
// simulateUpdate() isn't needed.
await Util.shortDelay();
// Next, wait for the second appendBuffer(), falling through to normal
// operations.
await simulateUpdate();
// Lastly, wait for the function-scoped MediaSourceEngine#appendBuffer()
// promise to resolve.
await appendVideo;
expect(videoSourceBuffer.abort).toHaveBeenCalledTimes(2);
});
});

describe('remove', () => {
Expand Down
3 changes: 1 addition & 2 deletions test/media/streaming_engine_unit.js
View file
Expand Up @@ -527,8 +527,7 @@ describe('StreamingEngine', () => {

expect(mediaSourceEngine.init).toHaveBeenCalledWith(expectedMseInit,
/** sequenceMode= */ false, /** manifestType= */ 'UNKNOWN',
/** ignoreManifestTimestampsInSegmentsMode= */ false,
/** hasTextStreams= */ true);
/** ignoreManifestTimestampsInSegmentsMode= */ false);
expect(mediaSourceEngine.init).toHaveBeenCalledTimes(1);

expect(mediaSourceEngine.setDuration).toHaveBeenCalledTimes(1);
Expand Down

0 comments on commit 80cacf6

Please sign in to comment.