Skip to content

Commit

Permalink
Merge pull request videojs#2 from ldayananda/partial-append-captions
Browse files Browse the repository at this point in the history 
feat(captions): allow partial appends of caption and ID3 data
  • Loading branch information
@gesinger
gesinger committed Sep 6, 2018
2 parents d43ba1c + d75a2d0 commit 1d06551
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 132 deletions.
28 changes: 11 additions & 17 deletions docs/lhls/new-segment-loader-sequence.plantuml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,18 @@ group Request
start/end times.
end note

"mux.js" -> "segment-transmuxer" : postMessage(...caption...)
"segment-transmuxer" -> "media-segment-request" : onCaptions(...)
"media-segment-request" -> SegmentLoader : captionsFn(...)
note left
Gets captions from transmux.
end note

"mux.js" -> "segment-transmuxer" : postMessage(...id3Frame...)
"segment-transmuxer" -> "media-segment-request" : onId3(...)
"media-segment-request" -> SegmentLoader : id3Fn(...)
note left #moccasin
Passes along metadata for
partial transmux.
note left
Gets metadata from transmux.
end note

"mux.js" -> "segment-transmuxer" : postMessage(...data...)
Expand All @@ -89,20 +95,8 @@ group Request

"mux.js" -> "segment-transmuxer" : postMessage(...done...)
note left
Gathers GOP info, captions, and
metadata, and calls done callback.
end note
"segment-transmuxer" -> "media-segment-request" : onId3(...)
"media-segment-request" -> SegmentLoader : id3Fn(...)
note left
Passes along metadata for full
segment transmux.
end note
"segment-transmuxer" -> "media-segment-request" : onCaptions(...)
"media-segment-request" -> SegmentLoader : captionsFn(...)
note left
Passes along captions for full
segment transmux.
Gathers GOP info, and calls
done callback.
end note
"segment-transmuxer" -> "media-segment-request" : onDone(...)
"media-segment-request" -> SegmentLoader : doneFn(...)
Expand Down
Binary file modified docs/lhls/new-segment-loader-sequence.plantuml.png
View file
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
18 changes: 10 additions & 8 deletions src/media-segment-request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ const handleKeyResponse = (segment, finishProcessingFn) => (error, request) => {
* this request
*/
const handleInitSegmentResponse =
(segment, captionParser, finishProcessingFn) => (error, request) => {
({segment, captionParser, finishProcessingFn}) => (error, request) => {
const response = request.response;
const errorObj = handleErrors(error, request);

Expand Down Expand Up @@ -336,8 +336,8 @@ const transmuxAndNotify = ({
onId3: (id3Frames, dispatchType) => {
id3Fn(segment, id3Frames, dispatchType);
},
onCaptions: (captions, captionStreams) => {
captionsFn(segment, captions, captionStreams);
onCaptions: (captions) => {
captionsFn(segment, [captions]);
},
onDone: (result) => {
// To handle partial appends, there won't be a done function passed in (since
Expand Down Expand Up @@ -397,8 +397,8 @@ const handleSegmentBytes = ({
segment.map.videoTrackIds,
segment.map.timescales);

if (parsed && parsed.captions) {
captionsFn(segment, parsed.captions, parsed.captionStreams);
if (parsed && parsed.captions && parsed.captions.length > 0) {
captionsFn(segment, parsed.captions);
}

doneFn(null, segment, {});
Expand Down Expand Up @@ -741,9 +741,11 @@ export const mediaSegmentRequest = ({
responseType: 'arraybuffer',
headers: segmentXhrHeaders(segment.map)
});
const initSegmentRequestCallback = handleInitSegmentResponse(segment,
captionParser,
finishProcessingFn);
const initSegmentRequestCallback = handleInitSegmentResponse({
segment,
captionParser,
finishProcessingFn
});
const initSegmentXhr = xhr(initSegmentOptions, initSegmentRequestCallback);

activeXhrs.push(initSegmentXhr);
Expand Down
82 changes: 53 additions & 29 deletions src/segment-loader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,10 @@ export default class SegmentLoader extends videojs.EventTarget {
video: null
};
this.callQueue_ = [];
this.id3Queue_ = [];
this.metadataQueue_ = {
id3: [],
caption: []
};

// Fragmented mp4 playback
this.activeInitSegmentId_ = null;
Expand Down Expand Up @@ -357,7 +360,8 @@ export default class SegmentLoader extends videojs.EventTarget {
// clear out the segment being processed
this.pendingSegment_ = null;
this.callQueue_ = [];
this.id3Queue_ = [];
this.metadataQueue_.id3 = [];
this.metadataQueue_.caption = [];
}

checkForAbort_(requestId) {
Expand Down Expand Up @@ -643,7 +647,8 @@ export default class SegmentLoader extends videojs.EventTarget {
this.mediaIndex = null;
this.syncPoint_ = null;
this.callQueue_ = [];
this.id3Queue_ = [];
this.metadataQueue_.id3 = [];
this.metadataQueue_.caption = [];
this.abort();
}

Expand Down Expand Up @@ -1138,20 +1143,43 @@ export default class SegmentLoader extends videojs.EventTarget {
}
}

handleCaptions_(simpleSegment, captions, captionStreams) {
// Don't need to check for abort since captions are only handled for non partial
// appends at the moment (therefore, they will only trigger once a segment is finished
// being transmuxed).
createCaptionsTrackIfNotExists(
this.inbandTextTracks_, this.hls_.tech_, captionStreams);
addCaptionData({
captionArray: captions,
inbandTextTracks: this.inbandTextTracks_,
// full segments appends already offset times in the transmuxer
timestampOffset: 0
handleCaptions_(simpleSegment, captions) {
if (this.checkForAbort_(simpleSegment.requestId) ||
this.abortRequestEarly_(simpleSegment.stats)) {
return;
}

// This could only happen with fmp4 segments, but
// should still not happen in general
if (captions.length === 0) {
this.logger_('SegmentLoader received no captions from a caption event');
return;
}

const segmentInfo = this.pendingSegment_;

// Wait until we have some video data so that caption timing
// can be adjusted by the timestamp offset
if (!segmentInfo.hasAppendedData_) {
this.metadataQueue_.caption.push(
this.handleCaptions_.bind(this, simpleSegment, captions));
return;
}

const timestampOffset = this.sourceUpdater_.videoTimestampOffset() === null ?
this.sourceUpdater_.audioTimestampOffset() :
this.sourceUpdater_.videoTimestampOffset();

captions.forEach((caption) => {
createCaptionsTrackIfNotExists(
this.inbandTextTracks_, this.hls_.tech_, caption.stream);
addCaptionData({
captionArray: [caption],
inbandTextTracks: this.inbandTextTracks_,
timestampOffset
});
});

// TODO
// Reset stored captions since we added parsed
// captions to a text track at this point
this.captionParser_.clearParsedCaptions();
Expand All @@ -1167,19 +1195,14 @@ export default class SegmentLoader extends videojs.EventTarget {

// we need to have appended data in order for the timestamp offset to be set
if (!segmentInfo.hasAppendedData_) {
this.id3Queue_.push(
this.metadataQueue_.id3.push(
this.handleId3_.bind(this, simpleSegment, id3Frames, dispatchType));
return;
}

// full segments appends already offset times in the transmuxer
let timestampOffset = 0;

if (this.handlePartialData_) {
timestampOffset = this.sourceUpdater_.videoTimestampOffset() === null ?
this.sourceUpdater_.videoTimestampOffset() :
this.sourceUpdater_.audioTimestampOffset();
}
const timestampOffset = this.sourceUpdater_.videoTimestampOffset() === null ?
this.sourceUpdater_.audioTimestampOffset() :
this.sourceUpdater_.videoTimestampOffset();

// There's potentially an issue where we could double add metadata if there's a muxed
// audio/video source with a metadata track, and an alt audio with a metadata track.
Expand All @@ -1193,11 +1216,12 @@ export default class SegmentLoader extends videojs.EventTarget {
});
}

processId3Queue_(simpleSegment) {
const id3Queue = this.id3Queue_;
processMetadataQueue_() {
this.metadataQueue_.id3.forEach((fn) => fn());
this.metadataQueue_.caption.forEach((fn) => fn());

this.id3Queue_ = [];
id3Queue.forEach((fun) => fun());
this.metadataQueue_.id3 = [];
this.metadataQueue_.caption = [];
}

processCallQueue_() {
Expand Down Expand Up @@ -1321,7 +1345,7 @@ export default class SegmentLoader extends videojs.EventTarget {
// has had a chance to be set.
segmentInfo.hasAppendedData_ = true;
// Now that the timestamp offset should be set, we can append any waiting ID3 tags.
this.processId3Queue_(simpleSegment);
this.processMetadataQueue_();

this.appendData_(segmentInfo, result);
}
Expand Down
57 changes: 8 additions & 49 deletions src/segment-transmuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,56 +47,16 @@ export const handleData_ = (event, transmuxedData, callback) => {
};

export const handleDone_ = ({
event,
transmuxedData,
onId3,
onCaptions,
callback
}) => {
// all buffers should have been flushed from the muxer, so start processing anything we
// have received
let sortedSegments = {
captions: [],
gopInfo: transmuxedData.gopInfo,
videoTimingInfo: transmuxedData.videoTimingInfo,
audioTimingInfo: transmuxedData.audioTimingInfo,
captionStreams: {}
};
const buffer = transmuxedData.buffer;
let metadata = [];
let captions = [];
let captionStreams = [];

// Previously we only returned data on data events,
// not on done events. Clear out the buffer to keep that consistent.
transmuxedData.buffer = [];

// Sort segments into separate video/audio arrays and
// keep track of their total byte lengths
sortedSegments = buffer.reduce((segmentObj, segment) => {
// Gather any captions into a single array
if (segment.captions) {
captions = captions.concat(segment.captions);
}

// Gather any metadata into a single array
if (segment.metadata) {
metadata = metadata.concat(segment.metadata);
}

if (segment.captionStreams) {
captionStreams = videojs.mergeOptions(captionStreams, segment.captionStreams);
}

return segmentObj;
}, sortedSegments);

if (metadata && metadata.length) {
onId3(metadata, metadata.dispatchType);
}
if (captions && captions.length) {
onCaptions(captions, captionStreams);
}

callback(sortedSegments);
// all buffers should have been flushed from the muxer, so start processing anything we
// have received
callback(transmuxedData);
};

export const handleGopInfo_ = (event, transmuxedData) => {
Expand Down Expand Up @@ -143,10 +103,12 @@ export const processTransmux = ({
if (event.data.action === 'videoTimingInfo') {
onVideoTimingInfo(event.data.videoTimingInfo);
}
// only used for partial transmuxer, full transmuxer will handle on done
if (event.data.action === 'id3Frame') {
onId3([event.data.id3Frame], event.data.id3Frame.dispatchType);
}
if (event.data.action === 'caption') {
onCaptions(event.data.caption);
}

// wait for the transmuxed event since we may have audio and video
if (event.data.type !== 'transmuxed') {
Expand All @@ -155,10 +117,7 @@ export const processTransmux = ({

transmuxer.removeEventListener('message', handleMessage);
handleDone_({
event,
transmuxedData,
onId3,
onCaptions,
callback: onDone
});

Expand Down
31 changes: 23 additions & 8 deletions src/transmuxer-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,19 @@ const wireFullTransmuxerEvents = function(transmuxer) {
}, [segment.data]);
});

if (transmuxer.captionStream) {
transmuxer.captionStream.on('data', function(caption) {
window.postMessage({
action: 'caption',
data: caption
});
transmuxer.on('id3Frame', function(id3Frame) {
window.postMessage({
action: 'id3Frame',
id3Frame
});
}
});

transmuxer.on('caption', function(caption) {
window.postMessage({
action: 'caption',
caption
});
});

transmuxer.on('done', function(data) {
window.postMessage({ action: 'done' });
Expand All @@ -83,7 +88,10 @@ const wireFullTransmuxerEvents = function(transmuxer) {
});

transmuxer.on('trackinfo', function(trackInfo) {
window.postMessage({ action: 'trackinfo', trackInfo });
window.postMessage({
action: 'trackinfo',
trackInfo
});
});

transmuxer.on('audioTimingInfo', function(audioTimingInfo) {
Expand Down Expand Up @@ -149,6 +157,13 @@ const wirePartialTransmuxerEvents = function(transmuxer) {
});
});

transmuxer.on('caption', function(caption) {
window.postMessage({
action: 'caption',
caption
});
});

transmuxer.on('done', function(data) {
window.postMessage({
action: 'done',
Expand Down

0 comments on commit 1d06551

Please sign in to comment.